diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 5eaf876e3b2..02472cdbfff 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -8,6 +8,8 @@ For string-valued enums and their Serde conversions, follow [.agents/skills/rust For error types, error variants and conversions between crate errors, follow [.agents/skills/rust-error/SKILL.md](.agents/skills/rust-error/SKILL.md) +The process-wide rustls `CryptoProvider` (ring) is installed once by the host. rust-proxy does it at the top of `main` through `litellm_gateway::install_crypto_provider`, and python-bridge must do the same in its module init before it builds any rustls config. Library crates never install it. The AWS SDK links aws-lc-rs next to ring, so without an installed default the first rustls config a library builds, such as redis-rs TLS, panics + For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of a hand-listed `ALL` array. Derive Strum string conversions instead of hand-written variant-to-string matches Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all` diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index bfbe543e235..b139f959e3e 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3703,7 +3703,10 @@ dependencies = [ name = "litellm-config" version = "0.1.0" dependencies = [ + "litellm-core-utils", "litellm-error", + "litellm-python-compat", + "litellm-redis-types", "litellm-router-types", "litellm-secret-value", "rstest", @@ -3714,7 +3717,8 @@ dependencies = [ "tempfile", "thiserror 2.0.19", "tracing", - "tracing-subscriber", + "url", + "veil", ] [[package]] @@ -3817,6 +3821,7 @@ dependencies = [ "http-body-util", "litellm-circuit-breaker", "litellm-config", + "litellm-core-utils", "litellm-error", "litellm-gateway-auth", "litellm-gateway-inference", @@ -3826,11 +3831,14 @@ dependencies = [ "litellm-inference", "litellm-llms", "litellm-redis-client", + "litellm-redis-connect", "litellm-redis-testing", + "litellm-redis-types", "litellm-secret-value", "litellm-secrets", "litellm-tracing", "rstest", + "rustls 0.23.42", "serde", "serde_json", "tempfile", @@ -4415,23 +4423,53 @@ dependencies = [ "futures-util", "litellm-circuit-breaker", "litellm-redis-testing", + "litellm-redis-types", "litellm-secret-value", + "pem", "redis", "rstest", + "rustls 0.23.42", + "rustls-native-certs", "socket2 0.6.5", "thiserror 2.0.19", "tokio", ] +[[package]] +name = "litellm-redis-connect" +version = "0.1.0" +dependencies = [ + "litellm-circuit-breaker", + "litellm-config", + "litellm-redis-client", + "litellm-redis-testing", + "litellm-redis-types", + "rstest", + "rustls 0.23.42", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tracing", +] + [[package]] name = "litellm-redis-testing" version = "0.1.0" dependencies = [ + "rcgen", + "tempfile", "testcontainers-modules", "thiserror 2.0.19", "tokio", ] +[[package]] +name = "litellm-redis-types" +version = "0.1.0" +dependencies = [ + "litellm-secret-value", +] + [[package]] name = "litellm-router" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 75cac2a5d63..76a588b6d9c 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -73,7 +73,9 @@ litellm-host-python = { path = "crates/host-python" } litellm-python-compat = { path = "crates/python-compat" } litellm-circuit-breaker = { path = "crates/circuit-breaker" } litellm-redis-client = { path = "crates/redis-client" } +litellm-redis-connect = { path = "crates/redis-connect" } litellm-redis-testing = { path = "crates/redis-testing" } +litellm-redis-types = { path = "crates/redis-types" } litellm-cost = { path = "crates/cost" } litellm-model-catalog = { path = "crates/model-catalog" } litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" } diff --git a/litellm-rust/crates/config/AGENTS.md b/litellm-rust/crates/config/AGENTS.md new file mode 100644 index 00000000000..51a6e095eec --- /dev/null +++ b/litellm-rust/crates/config/AGENTS.md @@ -0,0 +1,22 @@ +## Redis settings + +- The boundary is surface in, `ResolvedRedis` out. A surface turns what it was given into a `ConfiguredRedis`, the redis-py kwargs `_get_redis_client_logic` would receive: `RedisSettings::coordination` for `general_settings.coordination_redis`, `RedisSettings::cache_params` for `litellm_settings.cache_params`, and `ConfiguredRedis::redis_cache_kwargs` for a `RedisCache.redis_kwargs` dict. `ConfiguredRedis::resolve` is then the same for every host and returns a `Resolution`: the `litellm_redis_types::ResolvedRedis` plus every accepted-but-ignored setting by source, which the host logs. A new surface is a new constructor, never a flag on `resolve` +- Resolution runs in three steps, one file each. `layers.rs` puts the configured kwargs over their `REDIS_*` values. `options.rs` reads the result into the typed `Options`, rejects unsupported settings, collects ignored ones and fills Python's `get_secret` fallbacks. `endpoint.rs` and `auth.rs` pick the target and the credentials from `Options` alone +- `resolve` follows `litellm/_redis.py` `_get_redis_client_logic` plus `_credential_provider_auth_kwargs` and the per-client allowlists of `get_redis_async_client` (redis-py 5.3.1). Python and Rust pods share one Redis during the switch, so a change there is a change to a wire contract +- That contract is `resolves_like_python_redis` in `tests/redis.rs` and its mirror `test_redis_settings_resolve_like_the_rust_resolver_expects` in `tests/unit/test_redis.py`. Both hold the same named cases with the expected view written next to the input, the Python side reading the effective redis-py 5.3.1 connection objects without connecting. A new resolution rule is a new case in both files, never a generated fixture. A case where the two sides expect different outcomes is a divergence and is listed below +- `resolve` takes a `SecretSnapshot` and an environment `Lookup`. The snapshot holds every name `ConfiguredRedis::secret_names` lists, read the way Python's `get_secret` reads it (secret manager, then environment). The host fills it once before resolving, python-bridge through `get_secret` and rust-proxy from its environment, so the resolver never calls back into the host. The environment lookup is only for what Python reads straight from `os.environ` (`REDIS_USERNAME` for Azure). Reading a name the snapshot does not list fails a debug assertion +- Every Redis setting Python reads is one row in `SETTINGS` in `src/redis/keys.rs`: how Rust handles it (read, ignore, warn, reject), whether `_get_redis_env_kwarg_mapping` reads it as `REDIS_`, and the names `_get_redis_client_logic` falls back to. The env layer, the rejections, the ignored list and `secret_names` all come from that table. A read row is also a field of `Options` +- Lookups go through `litellm_core_utils::settings::Lookup`, never a closure type of our own +- A url carries its own TLS. redis-py's url allowlist is the plain `Connection` signature, so every `ssl_*` setting outside the url is dropped, and a credential provider strips the url's userinfo. A `rediss://` url's `ssl_cert_reqs`, `ssl_ca_certs`, `ssl_certfile`, `ssl_keyfile` and `ssl_check_hostname` query parameters are read like `parse_url` reads them +- TLS options and `db` are only read for the endpoint that uses them, so a bad `ssl_cert_reqs` never fails a url target. A cluster with a non-zero `db`, and Sentinel with a `host` or `port`, are errors because redis-py raises building those clients +- An empty value (`""`, `[]`, `{}`) is the same as an unset one, so it falls through to its environment fallback the way Python's `x or get_secret(...)` does. Nulls depend on the surface. `coordination_redis` drops every null like `model_dump(exclude_none=True)`, `cache_params` keeps a null as an explicit `None` that hides the `REDIS_*` value unless `Cache` or `RedisCache` names that parameter, and `RedisCache.redis_kwargs` keeps every null. A missing `os.environ/` reference is a null, as `load_config` resolves it +- Precedence matches Python. The target is cluster, then URL, then Sentinel (`sentinel_nodes` plus `service_name`), then host. Auth is an explicit `credential_provider`, then GCP, then Azure, then AWS, and the first one wins with a warning +- Numbers are read with `litellm_python_compat::number`, so whitespace and digit underscores parse like Python's `int()` and `float()`. A value they cannot parse (`port: abc`, `socket_timeout: slow`) is dropped with a warning and the default is used, like `_coerce_redis_kwargs_types`. A number that parses but is out of range (negative seconds, a port above 65535) fails with `RedisError::Invalid` +- Flags (`ssl`, `ssl_check_hostname`, `aws_iam_auth`, `aws_iam_serverless`) are read one way on every surface. `true`, `1` and `yes` are true and `false`, `0`, `no`, `off`, `f` and `n` are false, the text redis-py's `_str_to_bool` and the proxy's pydantic models agree on. Any other text is `RedisError::Invalid`, so a TLS setting is never guessed +- `RedisSettings::coordination` mirrors the proxy's `CoordinationRedisParams` pydantic model, which rejects a malformed `port` before redis-py coerces anything +- Known divergences, kept on purpose. Each is a case in both contract tests with a different expected outcome on each side + - A url query parameter other than the TLS ones, or any query parameter on a plain `redis://` url, is `RedisError::Unsupported` naming `url ?`. redis-py turns each one into a client kwarg + - Settings that would change the server, identity or TLS trust if ignored are `RedisError::Unsupported` naming the setting, including `ssl_ciphers`, `ssl_min_version`, `ssl_password`, `ssl_ca_data`, OCSP options, `unix_socket_path`, `connection_pool`, `event_dispatcher`, `redis_connect_func`, `retry` + - `ssl_ca_path` is rejected on both sides. redis-py 5.3.1's Redis constructor raises TypeError and Rust returns Unsupported naming the setting + - `retry_on_timeout`, `retry_on_error`, `protocol`, `cache`, `cache_config`, `health_check_interval`, `socket_keepalive` and `max_connections` are accepted with a warning and ignored. Cluster clients still turn TCP keepalive on, like Python's async cluster default, but redis-rs has no periodic health check, so Python's 25s cluster default has no equivalent + - A boolean `REDIS_RETRY_ON_ERROR` reaches redis-py's connection constructor as a non-iterable and raises TypeError. Rust ignores it with a warning and otherwise resolves the default host connection, verified against the host-default fixture diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml index 278aea6200e..b33472e14c4 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/config/Cargo.toml @@ -7,16 +7,20 @@ repository.workspace = true [dependencies] litellm-secret-value.workspace = true +litellm-core-utils.workspace = true litellm-router-types.workspace = true +litellm-redis-types.workspace = true +litellm-python-compat.workspace = true serde.workspace = true serde_json.workspace = true serde_yaml_ng = "0.10.0" strum.workspace = true thiserror.workspace = true tracing.workspace = true +url.workspace = true +veil.workspace = true [dev-dependencies] litellm-error.workspace = true rstest.workspace = true tempfile.workspace = true -tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs index e5219ac992c..300f15380bd 100644 --- a/litellm-rust/crates/config/src/error.rs +++ b/litellm-rust/crates/config/src/error.rs @@ -11,11 +11,4 @@ pub enum Error { path: PathBuf, source: serde_yaml_ng::Error, }, - #[error("Redis host does not yet support configured setting {0}")] - RedisSetting(String), - #[error("invalid Redis setting {setting}: {reason}")] - RedisInvalid { - setting: &'static str, - reason: &'static str, - }, } diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs index b8af6f11d2e..1e253a11076 100644 --- a/litellm-rust/crates/config/src/lib.rs +++ b/litellm-rust/crates/config/src/lib.rs @@ -15,8 +15,8 @@ pub use litellm_router_types::{LitellmParams, Spelled}; pub use mcp::{McpAuth, McpServer, McpTransport}; pub use model::Model; pub use redis::{ - CacheSettings, Lookup, RedisCluster, RedisEndpoint, RedisHost, RedisNode, RedisSettings, - ResolvedRedis, environment_has_target, + CacheSettings, ConfiguredRedis, Expected, RedisError, RedisSettings, Resolution, + SecretSnapshot, environment_has_target, }; pub use settings::{ ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings, diff --git a/litellm-rust/crates/config/src/redis.rs b/litellm-rust/crates/config/src/redis.rs index 7a91691c54b..c07217595c4 100644 --- a/litellm-rust/crates/config/src/redis.rs +++ b/litellm-rust/crates/config/src/redis.rs @@ -1,122 +1,25 @@ -use std::{collections::BTreeMap, fmt, time::Duration}; +use std::collections::BTreeMap; -use litellm_router_types::Spelled; -use litellm_secret_value::SecretValue; +use litellm_core_utils::settings::Lookup; +use litellm_redis_types::{Endpoint, RedisAuth, ResolvedRedis}; use serde::{Deserialize, Deserializer, de::Error as _}; +use veil::Redact; -use crate::{AdditionalFields, Error, Object, Value}; +use crate::{Object, Value}; -pub type Lookup<'a> = &'a dyn Fn(&str) -> Option; +mod auth; +mod endpoint; +mod error; +mod keys; +mod layers; +mod options; +mod secrets; +mod tls; -const SUPPORTED: &[&str] = &[ - "url", - "host", - "port", - "username", - "password", - "ssl", - "db", - "socket_timeout", - "startup_nodes", - "namespace", -]; +pub use error::{Expected, RedisError}; +pub use secrets::SecretSnapshot; -const IGNORED: &[&str] = &[ - "max_connections", - "single_connection_client", - "health_check_interval", - "socket_keepalive", - "socket_keepalive_options", - "decode_responses", - "encoding", - "encoding_errors", - "charset", - "errors", - "client_name", - "lib_name", - "lib_version", - "sentinel_password", - "gcp_ssl_ca_certs", - "azure_client_id", - "azure_tenant_id", - "azure_client_secret", - "aws_iam_user_name", - "aws_iam_cache_name", - "aws_iam_region", - "aws_iam_serverless", -]; - -const WARN_IGNORED: &[&str] = &[ - "max_connections", - "health_check_interval", - "socket_keepalive", - "socket_keepalive_options", -]; - -const REDIS_PY_KWARGS: &[&str] = &[ - "aws_iam_auth", - "aws_iam_cache_name", - "aws_iam_region", - "aws_iam_serverless", - "aws_iam_user_name", - "azure_client_id", - "azure_client_secret", - "azure_redis_ad_token", - "azure_tenant_id", - "cache", - "cache_config", - "charset", - "client_name", - "db", - "decode_responses", - "encoding", - "encoding_errors", - "errors", - "event_dispatcher", - "gcp_service_account", - "gcp_ssl_ca_certs", - "health_check_interval", - "host", - "lib_name", - "lib_version", - "max_connections", - "password", - "port", - "protocol", - "redis_connect_func", - "retry_on_error", - "retry_on_timeout", - "single_connection_client", - "socket_connect_timeout", - "socket_keepalive", - "socket_keepalive_options", - "socket_timeout", - "ssl", - "ssl_ca_certs", - "ssl_ca_data", - "ssl_ca_path", - "ssl_cert_reqs", - "ssl_certfile", - "ssl_check_hostname", - "ssl_ciphers", - "ssl_keyfile", - "ssl_min_version", - "ssl_ocsp_context", - "ssl_ocsp_expected_cert", - "ssl_password", - "ssl_validate_ocsp", - "ssl_validate_ocsp_stapled", - "unix_socket_path", - "url", - "username", -]; - -const UNSUPPORTED_EXTRA: &[&str] = &[ - "credential_provider", - "sentinel_nodes", - "retry", - "connection_pool", -]; +use options::Options; const CACHE_ONLY: &[&str] = &[ "type", @@ -130,67 +33,13 @@ const CACHE_ONLY: &[&str] = &[ "_backend", ]; -#[derive(Clone, Default)] +/// A Redis section as `config.yaml` spells it. Each surface reads it its own way, so it resolves +/// only through [`RedisSettings::coordination`] or [`RedisSettings::cache_params`]. +#[derive(Clone, Default, Deserialize, Redact)] +#[serde(transparent)] pub struct RedisSettings { - pub url: Option, - pub host: Option>, - pub port: Option>, - pub username: Option, - pub password: Option, - pub ssl: Option>, - pub db: Option>, - pub socket_timeout: Option>, - pub startup_nodes: Option, - pub namespace: Option>, - pub additional_fields: AdditionalFields, - unparsed: DeferredInputs, -} - -impl<'de> Deserialize<'de> for RedisSettings { - fn deserialize>(deserializer: D) -> Result { - let fields = BTreeMap::::deserialize(deserializer)?; - let scalar = |key: &str| fields.get(key).and_then(as_text); - Ok(Self { - url: scalar("url").map(SecretValue::new), - host: scalar("host").map(Spelled::Value), - port: spelled(scalar("port")), - username: scalar("username").map(SecretValue::new), - password: scalar("password").map(SecretValue::new), - ssl: spelled(scalar("ssl")), - db: spelled(scalar("db")), - socket_timeout: spelled(scalar("socket_timeout")), - startup_nodes: fields - .get("startup_nodes") - .filter(|value| !value.is_null()) - .cloned(), - namespace: scalar("namespace").map(Spelled::Value), - additional_fields: cloned(&fields, |key, _| !SUPPORTED.contains(&key)), - unparsed: deferred_inputs(&fields), - }) - } -} - -impl fmt::Debug for RedisSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RedisSettings") - .field("url", &self.url) - .field("host", &self.host) - .field("port", &self.port) - .field("username", &self.username) - .field("password", &self.password) - .field("ssl", &self.ssl) - .field("db", &self.db) - .field("socket_timeout", &self.socket_timeout) - .field( - "startup_nodes", - &self.startup_nodes.as_ref().map(|_| ""), - ) - .field("namespace", &self.namespace) - .field("additional_fields", &self.additional_fields.keys()) - .field("unparsed", &self.unparsed.keys()) - .finish() - } + #[redact(with = "[REDACTED]")] + fields: BTreeMap, } #[derive(Clone, Debug)] @@ -216,233 +65,123 @@ impl<'de> Deserialize<'de> for CacheSettings { } } -#[derive(Clone, PartialEq)] -pub struct ResolvedRedis { - pub endpoint: RedisEndpoint, - pub namespace: Option, - pub timeout: Option, +/// The redis-py kwargs one surface hands `_get_redis_client_logic`, before `REDIS_*` and secrets. +#[derive(Clone, Redact)] +pub struct ConfiguredRedis { + #[redact(with = "[REDACTED]")] + fields: BTreeMap, + nulls: Nulls, } -impl fmt::Debug for ResolvedRedis { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ResolvedRedis") - .field("endpoint", &self.endpoint) - .field("namespace", &self.namespace) - .field("timeout", &self.timeout) - .finish() +/// Whether a null setting reaches redis-py as an explicit `None`, which outranks its `REDIS_*` value. +#[derive(Clone, Copy, Debug)] +enum Nulls { + /// `coordination_redis` reaches redis-py through `model_dump(exclude_none=True)`. + Dropped, + /// `Cache` drops a null only for the parameters it names and passes any other key on. + KeptUnlessNamedByCache, + /// `RedisCache.redis_kwargs` already holds only the nulls redis-py receives. + Kept, +} + +/// A resolved connection, plus the settings it was given that Rust accepts without honoring. +#[derive(Debug)] +pub struct Resolution { + pub redis: ResolvedRedis, + /// Where each ignored setting came from: its key, or the environment variable that set it. + pub ignored: Vec, +} + +impl RedisSettings { + /// `general_settings.coordination_redis`, which the proxy validates as a pydantic model first. + /// An `os.environ/` port is only known after resolution, where it is read like any other port. + pub fn coordination(&self) -> Result { + if let Some(port) = self.fields.get("port").and_then(options::as_text) + && !port.starts_with("os.environ/") + && port.parse::().is_err() + { + return Err(RedisError::Invalid { + setting: "port".to_owned(), + expected: Expected::Integer, + }); + } + Ok(ConfiguredRedis { + fields: self.fields.clone(), + nulls: Nulls::Dropped, + }) } -} -#[derive(Clone, PartialEq)] -pub enum RedisEndpoint { - Url(SecretValue), - Host(RedisHost), - Cluster(RedisCluster), -} - -impl fmt::Debug for RedisEndpoint { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Url(_) => formatter.debug_tuple("Url").field(&"").finish(), - Self::Host(host) => host.fmt(formatter), - Self::Cluster(cluster) => cluster.fmt(formatter), + /// `litellm_settings.cache_params` with `type: redis`, as `litellm.Cache` hands it to `RedisCache`. + pub fn cache_params(&self) -> ConfiguredRedis { + let alias = self + .fields + .get("redis_startup_nodes") + .filter(|_| !self.fields.contains_key("startup_nodes")) + .map(|nodes| ("startup_nodes".to_owned(), nodes.clone())); + let fields = self + .fields + .iter() + .filter(|(key, _)| { + key.as_str() != "redis_startup_nodes" && !CACHE_ONLY.contains(&key.as_str()) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .chain(alias) + .collect(); + ConfiguredRedis { + fields, + nulls: Nulls::KeptUnlessNamedByCache, } } } -#[derive(Clone, Debug, PartialEq)] -pub struct RedisHost { - pub host: String, - pub port: u16, - pub username: Option, - pub password: Option, - pub tls: bool, - pub db: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct RedisCluster { - pub nodes: Vec, - pub username: Option, - pub password: Option, - pub tls: bool, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct RedisNode { - pub host: String, - pub port: u16, -} - -#[derive(Clone)] -struct Entry { - value: Value, - source: String, -} - -type Kwargs = BTreeMap; - -impl RedisSettings { - pub fn resolve(&self, lookup: Lookup<'_>) -> Result { - self.resolve_with(lookup, false) +impl ConfiguredRedis { + /// `RedisCache.redis_kwargs`, the dict every Python surface ends in and the one python-bridge + /// hands over. + pub fn redis_cache_kwargs(fields: BTreeMap) -> Self { + Self { + fields, + nulls: Nulls::Kept, + } } - pub fn resolve_cache(&self, lookup: Lookup<'_>) -> Result { - self.resolve_with(lookup, true) + /// No configured settings, so only `REDIS_*` and the other environment fallbacks apply. + pub fn environment_only() -> Self { + Self { + fields: BTreeMap::new(), + nulls: Nulls::Dropped, + } } - fn resolve_with(&self, lookup: Lookup<'_>, cache: bool) -> Result { - let block = self.kwargs(cache); - let block: Kwargs = block - .into_iter() - .filter_map(|(key, entry)| { - let value = environment_reference(&entry.value, lookup)?; - Some((key, Entry { value, ..entry })) - }) - .collect(); - let environment = environment_kwargs(lookup); - let names_a_target = ["host", "startup_nodes", "sentinel_nodes"] - .iter() - .any(|key| block.contains_key(*key)); - let drop_environment_url = names_a_target && !block.contains_key("url"); - let merged: Kwargs = environment - .into_iter() - .filter(|(key, _)| !(key == "url" && drop_environment_url)) - .chain(block) - .collect(); - reject_unsupported(&merged, lookup)?; - warn_ignored(&merged); - - let startup_nodes = match merged.get("startup_nodes") { - Some(entry) => Some(cluster_nodes(&entry.value)?), - None => lookup("REDIS_CLUSTER_NODES") - .map(|nodes| cluster_nodes(&Value::String(nodes))) - .transpose()?, - }; - let username = secret_value(&merged, "username"); - let password = secret_value(&merged, "password"); - let tls = bool_value(&merged, "ssl"); - let endpoint = match (startup_nodes, text_value(&merged, "url")) { - (Some(nodes), _) if !nodes.is_empty() => RedisEndpoint::Cluster(RedisCluster { - nodes, + pub fn resolve( + &self, + secrets: &SecretSnapshot, + environment: &dyn Lookup, + ) -> Result { + let kwargs = layers::merged(self, secrets); + let (options, ignored) = Options::read(kwargs, secrets)?; + let auth = auth::redis_auth(&options, environment)?; + let endpoint = endpoint::endpoint(&options, &auth)?; + let auth = match (&endpoint, auth) { + (Endpoint::Url { .. }, RedisAuth::Static { username, .. }) => RedisAuth::Static { username, - password, - tls, - }), - (_, Some(url)) => RedisEndpoint::Url(SecretValue::new(url)), - _ => RedisEndpoint::Host(RedisHost { - host: text_value(&merged, "host").ok_or(Error::RedisInvalid { - setting: "host", - reason: "either host or url must be specified for redis", - })?, - port: integer(&merged, "port")?.unwrap_or(6379), - username, - password, - tls, - db: integer(&merged, "db")?.unwrap_or(0), - }), + password: None, + }, + (_, auth) => auth, }; - let timeout = merged - .get("socket_timeout") - .and_then(|entry| as_text(&entry.value)) - .map(|seconds| { - seconds - .parse::() - .ok() - .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) - .ok_or(Error::RedisInvalid { - setting: "socket_timeout", - reason: "must be a non-negative number of seconds", - }) - }) - .transpose()?; - Ok(ResolvedRedis { - endpoint, - namespace: text_value(&merged, "namespace"), - timeout, + Ok(Resolution { + redis: ResolvedRedis { + endpoint, + auth, + namespace: options::text(options.namespace.as_ref()), + timeout: options::seconds(options.socket_timeout.as_ref())?, + connect_timeout: options::seconds(options.socket_connect_timeout.as_ref())?, + }, + ignored, }) } - - fn kwargs(&self, cache: bool) -> Kwargs { - let known = [ - ( - "url", - self.url.as_ref().map(|value| value.expose().to_owned()), - ), - ("host", spelled_text(self.host.as_ref())), - ("port", spelled_text(self.port.as_ref())), - ( - "username", - self.username - .as_ref() - .map(|value| value.expose().to_owned()), - ), - ( - "password", - self.password - .as_ref() - .map(|value| value.expose().to_owned()), - ), - ("ssl", spelled_text(self.ssl.as_ref())), - ("db", spelled_text(self.db.as_ref())), - ("socket_timeout", spelled_text(self.socket_timeout.as_ref())), - ("namespace", spelled_text(self.namespace.as_ref())), - ]; - let known = known.into_iter().filter_map(|(key, value)| { - value.map(|value| { - ( - key.to_owned(), - Entry { - value: Value::String(value), - source: key.to_owned(), - }, - ) - }) - }); - let startup_nodes = self.startup_nodes.clone().map(|value| { - ( - "startup_nodes".to_owned(), - Entry { - value, - source: "startup_nodes".to_owned(), - }, - ) - }); - let extras = self.additional_fields.iter().filter_map(|(key, value)| { - if cache && CACHE_ONLY.contains(&key.as_str()) { - return None; - } - let key = match (cache, key.as_str()) { - (true, "redis_startup_nodes") => "startup_nodes".to_owned(), - _ => key.clone(), - }; - Some(( - key.clone(), - Entry { - value: value.clone(), - source: key, - }, - )) - }); - let unparsed = self.unparsed.iter().map(|(key, value)| { - ( - key.clone(), - Entry { - value: value.0.clone(), - source: key.clone(), - }, - ) - }); - extras - .chain(unparsed) - .chain(known) - .chain(startup_nodes) - .collect() - } } -pub fn environment_has_target(lookup: Lookup<'_>) -> bool { +pub fn environment_has_target(lookup: &dyn Lookup) -> bool { [ "REDIS_HOST", "REDIS_URL", @@ -450,218 +189,31 @@ pub fn environment_has_target(lookup: Lookup<'_>) -> bool { "REDIS_SENTINEL_NODES", ] .iter() - .any(|name| lookup(name).is_some()) + .any(|name| lookup.get(name).is_some()) } -fn reject_unsupported(merged: &Kwargs, lookup: Lookup<'_>) -> Result<(), Error> { - let active_auth = [ - merged - .get("gcp_service_account") - .map(|entry| entry.source.as_str()) - .or_else(|| lookup("REDIS_GCP_SERVICE_ACCOUNT").map(|_| "REDIS_GCP_SERVICE_ACCOUNT")), - text_value(merged, "azure_redis_ad_token") - .or_else(|| lookup("REDIS_AZURE_AD_TOKEN")) - .filter(|value| value.eq_ignore_ascii_case("true")) - .map(|_| { - merged - .get("azure_redis_ad_token") - .map_or("REDIS_AZURE_AD_TOKEN", |entry| entry.source.as_str()) - }), - bool_value(merged, "aws_iam_auth").then(|| merged["aws_iam_auth"].source.as_str()), - merged - .get("sentinel_nodes") - .map(|entry| entry.source.as_str()) - .or_else(|| lookup("REDIS_SENTINEL_NODES").map(|_| "REDIS_SENTINEL_NODES")), - ]; - if let Some(setting) = active_auth.into_iter().flatten().next() { - return Err(Error::RedisSetting(setting.to_owned())); - } - match merged - .iter() - .find(|(key, _)| is_unsupported(key)) - .map(|(_, entry)| entry.source.clone()) - { - Some(setting) => Err(Error::RedisSetting(setting)), - None => Ok(()), +#[cfg(test)] +mod tests { + use super::RedisSettings; + use rstest::rstest; + + #[rstest] + #[case::scalar(serde_json::json!("credential-value"))] + #[case::object(serde_json::json!({"secret": "credential-value"}))] + #[case::array(serde_json::json!(["credential-value"]))] + fn credentials_are_redacted( + #[case] password: serde_json::Value, + #[values(false, true)] configured: bool, + ) { + let settings: RedisSettings = serde_json::from_value(serde_json::json!({ + "host": "redis", "password": password, + })) + .unwrap(); + let debug = match configured { + false => format!("{settings:?}"), + true => format!("{:?}", settings.cache_params()), + }; + assert!(!debug.contains("credential-value")); + assert!(debug.contains("[REDACTED]")); } } - -fn warn_ignored(merged: &Kwargs) { - for setting in WARN_IGNORED { - if merged.contains_key(*setting) { - tracing::warn!(setting, "Redis setting is accepted but ignored"); - } - } -} - -fn is_unsupported(key: &str) -> bool { - let known = REDIS_PY_KWARGS.contains(&key) || UNSUPPORTED_EXTRA.contains(&key); - let handled = SUPPORTED.contains(&key) - || IGNORED.contains(&key) - || matches!( - key, - "gcp_service_account" | "azure_redis_ad_token" | "aws_iam_auth" - ); - known && !handled -} - -fn environment_kwargs(lookup: Lookup<'_>) -> Kwargs { - REDIS_PY_KWARGS - .iter() - .filter_map(|key| { - let source = format!("REDIS_{}", key.to_ascii_uppercase()); - let value = lookup(&source)?; - Some(( - (*key).to_owned(), - Entry { - value: Value::String(value), - source, - }, - )) - }) - .collect() -} - -fn environment_reference(value: &Value, lookup: Lookup<'_>) -> Option { - match value { - Value::String(text) => match text.strip_prefix("os.environ/") { - Some(name) => lookup(name).map(Value::String), - None => Some(value.clone()), - }, - Value::Null => None, - _ => Some(value.clone()), - } -} - -fn as_text(value: &Value) -> Option { - match value { - Value::String(text) => Some(text.clone()), - Value::Number(number) => Some(number.to_string()), - Value::Bool(flag) => Some(flag.to_string()), - _ => None, - } -} - -fn text_value(kwargs: &Kwargs, key: &str) -> Option { - kwargs.get(key).and_then(|entry| as_text(&entry.value)) -} - -fn secret_value(kwargs: &Kwargs, key: &str) -> Option { - text_value(kwargs, key).map(SecretValue::new) -} - -fn bool_value(kwargs: &Kwargs, key: &str) -> bool { - match kwargs.get(key).map(|entry| &entry.value) { - Some(Value::Bool(flag)) => *flag, - Some(value) => as_text(value) - .is_some_and(|text| matches!(text.to_ascii_lowercase().as_str(), "true" | "1" | "yes")), - None => false, - } -} - -fn integer(kwargs: &Kwargs, key: &'static str) -> Result, Error> { - kwargs - .get(key) - .and_then(|entry| as_text(&entry.value)) - .map(|text| { - text.parse().map_err(|_| Error::RedisInvalid { - setting: key, - reason: "must be an integer", - }) - }) - .transpose() -} - -const INVALID_NODES: Error = Error::RedisInvalid { - setting: "startup_nodes", - reason: "must be a list of {host, port} objects", -}; - -fn cluster_nodes(value: &Value) -> Result, Error> { - match value { - Value::String(json) => { - let nodes: Vec> = - serde_json::from_str(json).map_err(|_| INVALID_NODES)?; - nodes - .iter() - .map(|node| { - cluster_node( - node.get("host").and_then(json_text), - node.get("port").and_then(json_text), - ) - }) - .collect() - } - Value::Sequence(nodes) => nodes - .iter() - .map(|node| { - cluster_node( - node.get("host").and_then(as_text), - node.get("port").and_then(as_text), - ) - }) - .collect(), - _ => Err(INVALID_NODES), - } -} - -fn cluster_node(host: Option, port: Option) -> Result { - match (host, port.and_then(|port| port.parse().ok())) { - (Some(host), Some(port)) => Ok(RedisNode { host, port }), - _ => Err(INVALID_NODES), - } -} - -fn json_text(value: &serde_json::Value) -> Option { - match value { - serde_json::Value::String(text) => Some(text.clone()), - serde_json::Value::Number(number) => Some(number.to_string()), - _ => None, - } -} - -fn spelled_text(value: Option<&Spelled>) -> Option { - value.map(|value| match value { - Spelled::Value(value) => value.to_string(), - Spelled::Text(value) => value.clone(), - }) -} - -fn spelled(text: Option) -> Option> { - text.map(|text| match text.parse() { - Ok(value) => Spelled::Value(value), - Err(_) => Spelled::Text(text), - }) -} - -fn cloned( - fields: &BTreeMap, - keep: impl Fn(&str, &Value) -> bool, -) -> AdditionalFields { - fields - .iter() - .filter(|(key, value)| keep(key, value)) - .map(|(key, value)| (key.clone(), value.clone())) - .collect() -} - -#[derive(Clone)] -struct DeferredInput(Value); - -type DeferredInputs = BTreeMap; - -fn deferred_inputs(fields: &BTreeMap) -> DeferredInputs { - fields - .iter() - .filter(|(key, value)| is_unparsed(key, value)) - .map(|(key, value)| (key.clone(), DeferredInput(value.clone()))) - .collect() -} - -fn is_unparsed(key: &str, value: &Value) -> bool { - SUPPORTED.contains(&key) - && match key { - "startup_nodes" => value.is_null(), - _ => as_text(value).is_none(), - } -} diff --git a/litellm-rust/crates/config/src/redis/auth.rs b/litellm-rust/crates/config/src/redis/auth.rs new file mode 100644 index 00000000000..cb2ea0db53a --- /dev/null +++ b/litellm-rust/crates/config/src/redis/auth.rs @@ -0,0 +1,93 @@ +use litellm_core_utils::settings::Lookup; +use litellm_redis_types::{AwsIamAuth, AzureAdAuth, RedisAuth}; + +use super::{ + RedisError, + options::{self, AwsOptions, Options, flag, secret, text}, +}; + +pub(super) fn redis_auth( + options: &Options, + environment: &dyn Lookup, +) -> Result { + if options.credential_provider.is_some() { + return Ok(RedisAuth::Host); + } + let gcp = text(options.gcp.service_account.as_ref()); + let azure = text(options.azure.ad_token.as_ref()) + .is_some_and(|enabled| enabled.eq_ignore_ascii_case("true")); + let aws = flag(options.aws.auth.as_ref())?; + match (gcp, azure, aws) { + (Some(service_account), azure, aws) => { + if azure { + tracing::warn!( + "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. Using GCP IAM. Remove one to avoid misconfiguration." + ); + } + if aws { + tracing::warn!( + "Both GCP IAM (gcp_service_account) and AWS ElastiCache IAM (aws_iam_auth) are configured for Redis. Using GCP IAM. Remove one to avoid misconfiguration." + ); + } + Ok(RedisAuth::GcpIam { service_account }) + } + (None, true, aws) => { + if aws { + tracing::warn!( + "Both Azure AD (azure_redis_ad_token) and AWS ElastiCache IAM (aws_iam_auth) are configured for Redis. Using Azure AD. Remove one to avoid misconfiguration." + ); + } + Ok(RedisAuth::AzureAd(AzureAdAuth { + client_id: text(options.azure.client_id.as_ref()), + tenant_id: text(options.azure.tenant_id.as_ref()), + client_secret: secret(options.azure.client_secret.as_ref()), + username: environment + .get("REDIS_USERNAME") + .filter(|username| !username.is_empty()), + })) + } + (None, false, true) => aws_iam(&options.aws, uses_tls(options)?).map(RedisAuth::AwsIam), + (None, false, false) => Ok(RedisAuth::Static { + username: secret(options.username.as_ref()), + password: secret(options.password.as_ref()), + }), + } +} + +// _uses_tls: a url's scheme decides unless cluster nodes are set, otherwise the ssl kwarg does +fn uses_tls(options: &Options) -> Result { + match (&options.startup_nodes, text(options.url.as_ref())) { + (None, Some(url)) => Ok(url + .split_once("://") + .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("rediss"))), + _ => flag(options.ssl.as_ref()), + } +} + +fn aws_iam(aws: &AwsOptions, uses_tls: bool) -> Result { + if !uses_tls { + return Err(RedisError::AwsIamWithoutTls); + } + match ( + text(aws.user_name.as_ref()), + text(aws.cache_name.as_ref()), + text(aws.region.as_ref()), + ) { + (Some(user_name), Some(cache_name), Some(region)) => Ok(AwsIamAuth { + user_name, + cache_name, + region, + serverless: options::flag(aws.serverless.as_ref())?, + }), + (user_name, cache_name, region) => Err(RedisError::AwsIamIncomplete { + missing: [ + ("aws_iam_user_name", user_name.is_none()), + ("aws_iam_cache_name", cache_name.is_none()), + ("aws_iam_region", region.is_none()), + ] + .into_iter() + .filter_map(|(name, missing)| missing.then_some(name)) + .collect(), + }), + } +} diff --git a/litellm-rust/crates/config/src/redis/endpoint.rs b/litellm-rust/crates/config/src/redis/endpoint.rs new file mode 100644 index 00000000000..5e4f8b033a5 --- /dev/null +++ b/litellm-rust/crates/config/src/redis/endpoint.rs @@ -0,0 +1,156 @@ +use litellm_redis_types::{ + ClusterEndpoint, ClusterNode, Endpoint, HostEndpoint, RedisAuth, SentinelEndpoint, TlsOptions, +}; +use litellm_secret_value::SecretValue; + +use super::{ + Expected, RedisError, + layers::Entry, + options::{Options, as_text, flag, integer, invalid, secret, text}, + tls, +}; +use crate::Value; + +/// The target, first match wins like `get_redis_client`: cluster, url, Sentinel, then host. +pub(super) fn endpoint(options: &Options, auth: &RedisAuth) -> Result { + if let Some(nodes) = cluster_nodes(options.startup_nodes.as_ref())? { + reject_cluster_db(options.db.as_ref())?; + return Ok(Endpoint::Cluster(ClusterEndpoint { + nodes, + tls: tls_for(options, auth)?, + })); + } + if let Some(url) = text(options.url.as_ref()) { + return Ok(Endpoint::Url { + tls: tls::url_tls(&url)?, + url: SecretValue::new(match auth { + RedisAuth::Static { .. } => url, + _ => without_userinfo(&url), + }), + }); + } + if let Some(sentinels) = options.sentinel.nodes.as_ref() { + return sentinel(options, auth, sentinels).map(Endpoint::Sentinel); + } + Ok(Endpoint::Host(HostEndpoint { + host: text(options.host.as_ref()).ok_or(RedisError::NoTarget)?, + port: integer(options.port.as_ref())?.unwrap_or(6379), + tls: tls_for(options, auth)?, + db: db(options)?, + })) +} + +fn tls_for(options: &Options, auth: &RedisAuth) -> Result, RedisError> { + if !flag(options.ssl.as_ref())? { + return Ok(None); + } + let gcp_ca_certs = match auth { + RedisAuth::GcpIam { .. } => options.gcp.ssl_ca_certs.as_ref(), + _ => None, + }; + tls::tls_options(&options.tls, gcp_ca_certs).map(Some) +} + +fn db(options: &Options) -> Result { + integer(options.db.as_ref()).map(|db| db.unwrap_or(0)) +} + +fn sentinel( + options: &Options, + auth: &RedisAuth, + sentinels: &Entry, +) -> Result { + let nodes = sentinel_nodes(&sentinels.value) + .ok_or_else(|| invalid(&sentinels.source, Expected::SentinelNodes))?; + let service_name = + text(options.sentinel.service_name.as_ref()).ok_or(RedisError::NoSentinelService)?; + // redis.asyncio.Sentinel passes the connection kwargs to each monitor's Redis(host=..., port=...), so a + // host or port setting raises TypeError when litellm builds the client + if let Some(entry) = options.host.as_ref().or(options.port.as_ref()) { + return Err(RedisError::SentinelWithHost { + setting: entry.source.clone(), + }); + } + Ok(SentinelEndpoint { + sentinels: nodes, + service_name, + sentinel_password: secret(options.sentinel.password.as_ref()), + tls: tls_for(options, auth)?, + db: db(options)?, + }) +} + +fn cluster_nodes(entry: Option<&Entry>) -> Result>, RedisError> { + let Some(entry) = entry else { + return Ok(None); + }; + let nodes = nodes(&entry.value, |node| { + node_address( + node.get("host").and_then(as_text), + node.get("port").and_then(as_text), + ) + }) + .ok_or_else(|| invalid(&entry.source, Expected::StartupNodes))?; + Ok((!nodes.is_empty()).then_some(nodes)) +} + +fn sentinel_nodes(value: &Value) -> Option> { + let nodes = nodes(value, |pair| match pair { + Value::Sequence(pair) if pair.len() == 2 => { + node_address(as_text(&pair[0]), as_text(&pair[1])) + } + _ => None, + })?; + (!nodes.is_empty()).then_some(nodes) +} + +fn nodes(value: &Value, parse: impl Fn(&Value) -> Option) -> Option> { + match value { + Value::String(json) => serde_json::from_str::>(json) + .ok()? + .iter() + .map(parse) + .collect(), + Value::Sequence(nodes) => nodes.iter().map(parse).collect(), + _ => None, + } +} + +fn node_address(host: Option, port: Option) -> Option { + Some(ClusterNode { + host: host?, + port: port?.parse().ok()?, + }) +} + +// redis-py's RedisCluster raises unless db is 0 or None +fn reject_cluster_db(db: Option<&Entry>) -> Result<(), RedisError> { + let Some(entry) = db else { + return Ok(()); + }; + let is_zero = match &entry.value { + Value::String(_) => matches!(integer::(Some(entry)), Ok(None | Some(0))), + Value::Bool(flag) => !flag, + Value::Number(number) => number.as_f64() == Some(0.0), + _ => false, + }; + match is_zero { + true => Ok(()), + false => Err(RedisError::DbInCluster { + setting: entry.source.clone(), + }), + } +} + +// _get_redis_url_kwargs keeps only plain Connection kwargs, so redis-py drops every ssl_* setting for a url +// and a credential provider replaces the url's userinfo, which _url_without_userinfo strips +fn without_userinfo(url: &str) -> String { + let Some((scheme, rest)) = url.split_once("://") else { + return url.to_owned(); + }; + let (authority, tail) = rest.split_at(rest.find(['/', '?', '#']).unwrap_or(rest.len())); + let host = authority + .rsplit_once('@') + .map_or(authority, |(_, host)| host); + format!("{scheme}://{host}{tail}") +} diff --git a/litellm-rust/crates/config/src/redis/error.rs b/litellm-rust/crates/config/src/redis/error.rs new file mode 100644 index 00000000000..b770f1ed33b --- /dev/null +++ b/litellm-rust/crates/config/src/redis/error.rs @@ -0,0 +1,35 @@ +#[derive(Debug, thiserror::Error)] +pub enum RedisError { + #[error("Redis host does not yet support configured setting {setting}")] + Unsupported { setting: String }, + #[error("invalid Redis setting {setting}: expected {expected}")] + Invalid { setting: String, expected: Expected }, + #[error("either host or url must be specified for redis")] + NoTarget, + #[error("service_name is required with sentinel_nodes")] + NoSentinelService, + #[error("{setting} cannot be combined with sentinel_nodes, which redis-py rejects at startup")] + SentinelWithHost { setting: String }, + #[error("{setting} must be 0 in cluster mode")] + DbInCluster { setting: String }, + #[error("AWS ElastiCache IAM Redis authentication requires TLS")] + AwsIamWithoutTls, + #[error("AWS ElastiCache IAM Redis authentication requires: {}", missing.join(", "))] + AwsIamIncomplete { missing: Vec<&'static str> }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +pub enum Expected { + #[strum(serialize = "an integer")] + Integer, + #[strum(serialize = "a boolean")] + Boolean, + #[strum(serialize = "a non-negative number of seconds")] + Seconds, + #[strum(serialize = "none, optional or required")] + CertRequirement, + #[strum(serialize = "a list of objects with host and port")] + StartupNodes, + #[strum(serialize = "a non-empty list of [host, port] pairs")] + SentinelNodes, +} diff --git a/litellm-rust/crates/config/src/redis/keys.rs b/litellm-rust/crates/config/src/redis/keys.rs new file mode 100644 index 00000000000..cdb19be12dd --- /dev/null +++ b/litellm-rust/crates/config/src/redis/keys.rs @@ -0,0 +1,181 @@ +use litellm_core_utils::settings::Lookup; + +use super::layers::Entry; +use crate::Value; + +// Every Redis setting litellm/_redis.py reads, with how Rust handles it and where else Python looks for it. +// environment marks the kwargs _get_redis_env_kwarg_mapping reads as REDIS_, and fallback lists the names +// _get_redis_client_logic consults with get_secret when the merged kwargs lack the setting +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Handling { + Read, + Ignore, + Warn, + Reject, +} + +pub(super) struct Setting { + pub(super) kwarg: &'static str, + pub(super) handling: Handling, + pub(super) environment: bool, + pub(super) fallback: &'static [&'static str], +} + +impl Setting { + const fn new( + kwarg: &'static str, + handling: Handling, + environment: bool, + fallback: &'static [&'static str], + ) -> Self { + Self { + kwarg, + handling, + environment, + fallback, + } + } + + pub(super) fn environment_name(&self) -> Option { + self.environment + .then(|| format!("REDIS_{}", self.kwarg.to_ascii_uppercase())) + } +} + +pub(super) const SETTINGS: &[Setting] = &[ + Setting::new("aws_iam_auth", Handling::Read, true, &[]), + Setting::new("aws_iam_cache_name", Handling::Read, true, &[]), + Setting::new( + "aws_iam_region", + Handling::Read, + true, + &["AWS_REGION", "AWS_DEFAULT_REGION"], + ), + Setting::new("aws_iam_serverless", Handling::Read, true, &[]), + Setting::new("aws_iam_user_name", Handling::Read, true, &[]), + Setting::new( + "azure_client_id", + Handling::Read, + true, + &["AZURE_CLIENT_ID"], + ), + Setting::new( + "azure_client_secret", + Handling::Read, + true, + &["AZURE_CLIENT_SECRET"], + ), + Setting::new( + "azure_redis_ad_token", + Handling::Read, + true, + &["REDIS_AZURE_AD_TOKEN"], + ), + Setting::new( + "azure_tenant_id", + Handling::Read, + true, + &["AZURE_TENANT_ID"], + ), + Setting::new("cache", Handling::Warn, true, &[]), + Setting::new("cache_config", Handling::Warn, true, &[]), + Setting::new("charset", Handling::Ignore, true, &[]), + Setting::new("client_name", Handling::Ignore, true, &[]), + Setting::new("connection_pool", Handling::Reject, false, &[]), + Setting::new("credential_provider", Handling::Read, false, &[]), + Setting::new("db", Handling::Read, true, &[]), + Setting::new("decode_responses", Handling::Ignore, true, &[]), + Setting::new("encoding", Handling::Ignore, true, &[]), + Setting::new("encoding_errors", Handling::Ignore, true, &[]), + Setting::new("errors", Handling::Ignore, true, &[]), + Setting::new("event_dispatcher", Handling::Reject, true, &[]), + Setting::new( + "gcp_service_account", + Handling::Read, + true, + &["REDIS_GCP_SERVICE_ACCOUNT"], + ), + Setting::new( + "gcp_ssl_ca_certs", + Handling::Read, + true, + &["REDIS_GCP_SSL_CA_CERTS"], + ), + Setting::new("health_check_interval", Handling::Warn, true, &[]), + Setting::new("host", Handling::Read, true, &[]), + Setting::new("lib_name", Handling::Ignore, true, &[]), + Setting::new("lib_version", Handling::Ignore, true, &[]), + Setting::new("max_connections", Handling::Warn, true, &[]), + Setting::new("namespace", Handling::Read, false, &[]), + Setting::new("password", Handling::Read, true, &[]), + Setting::new("port", Handling::Read, true, &[]), + Setting::new("protocol", Handling::Warn, true, &[]), + Setting::new("redis_connect_func", Handling::Reject, true, &[]), + Setting::new("retry", Handling::Reject, false, &[]), + Setting::new("retry_on_error", Handling::Warn, true, &[]), + Setting::new("retry_on_timeout", Handling::Warn, true, &[]), + Setting::new( + "sentinel_nodes", + Handling::Read, + false, + &["REDIS_SENTINEL_NODES"], + ), + Setting::new( + "sentinel_password", + Handling::Read, + false, + &["REDIS_SENTINEL_PASSWORD"], + ), + Setting::new( + "service_name", + Handling::Read, + false, + &["REDIS_SERVICE_NAME"], + ), + Setting::new("single_connection_client", Handling::Ignore, true, &[]), + Setting::new("socket_connect_timeout", Handling::Read, true, &[]), + Setting::new("socket_keepalive", Handling::Warn, true, &[]), + Setting::new("socket_keepalive_options", Handling::Warn, true, &[]), + Setting::new("socket_timeout", Handling::Read, true, &[]), + Setting::new("ssl", Handling::Read, true, &[]), + Setting::new("ssl_ca_certs", Handling::Read, true, &[]), + Setting::new("ssl_ca_data", Handling::Reject, true, &[]), + Setting::new("ssl_ca_path", Handling::Reject, true, &[]), + Setting::new("ssl_cert_reqs", Handling::Read, true, &[]), + Setting::new("ssl_certfile", Handling::Read, true, &[]), + Setting::new("ssl_check_hostname", Handling::Read, true, &[]), + Setting::new("ssl_ciphers", Handling::Reject, true, &[]), + Setting::new("ssl_keyfile", Handling::Read, true, &[]), + Setting::new("ssl_min_version", Handling::Reject, true, &[]), + Setting::new("ssl_ocsp_context", Handling::Reject, true, &[]), + Setting::new("ssl_ocsp_expected_cert", Handling::Reject, true, &[]), + Setting::new("ssl_password", Handling::Reject, true, &[]), + Setting::new("ssl_validate_ocsp", Handling::Reject, true, &[]), + Setting::new("ssl_validate_ocsp_stapled", Handling::Reject, true, &[]), + Setting::new( + "startup_nodes", + Handling::Read, + false, + &["REDIS_CLUSTER_NODES"], + ), + Setting::new("unix_socket_path", Handling::Reject, true, &[]), + Setting::new("url", Handling::Read, true, &[]), + Setting::new("username", Handling::Read, true, &[]), +]; + +pub(super) fn setting(kwarg: &str) -> Option<&'static Setting> { + SETTINGS.iter().find(|setting| setting.kwarg == kwarg) +} + +pub(super) fn fallback(kwarg: &str, lookup: &dyn Lookup) -> Option { + let names = setting(kwarg).map_or(&[][..], |setting| setting.fallback); + names.iter().find_map(|name| { + lookup + .get(name) + .filter(|value| !value.is_empty()) + .map(|value| Entry { + value: Value::String(value), + source: (*name).to_owned(), + }) + }) +} diff --git a/litellm-rust/crates/config/src/redis/layers.rs b/litellm-rust/crates/config/src/redis/layers.rs new file mode 100644 index 00000000000..02b54cae823 --- /dev/null +++ b/litellm-rust/crates/config/src/redis/layers.rs @@ -0,0 +1,87 @@ +use litellm_core_utils::settings::Lookup; + +use super::{ConfiguredRedis, Nulls, keys, options::Kwargs}; +use crate::Value; + +#[derive(Clone)] +pub(super) struct Entry { + pub(super) value: Value, + pub(super) source: String, +} + +const NAMED_BY_CACHE: &[&str] = &[ + "host", + "port", + "password", + "socket_timeout", + "startup_nodes", + "namespace", + "gcp_service_account", + "gcp_ssl_ca_certs", +]; + +/// The configured settings over their `REDIS_*` values, `{**environment_kwargs, **env_overrides}` +/// in `_get_redis_client_logic`. A null left in the result is an explicit `None` that hid its +/// environment value. +pub(super) fn merged(configured: &ConfiguredRedis, lookup: &dyn Lookup) -> Kwargs { + let block: Kwargs = configured + .fields + .iter() + .map(|(key, value)| (key, environment_reference(value, lookup))) + .filter(|(key, value)| !value.is_null() || keeps_null(configured.nulls, key)) + .map(|(key, value)| { + ( + key.clone(), + Entry { + value, + source: key.clone(), + }, + ) + }) + .collect(); + let names_a_target = ["host", "startup_nodes", "sentinel_nodes"] + .iter() + .any(|key| is_set(&block, key)); + let drop_environment_url = names_a_target && !is_set(&block, "url"); + environment(lookup) + .filter(|(key, _)| !(key == "url" && drop_environment_url)) + .chain(block) + .collect() +} + +fn keeps_null(nulls: Nulls, key: &str) -> bool { + match nulls { + Nulls::Dropped => false, + Nulls::KeptUnlessNamedByCache => !NAMED_BY_CACHE.contains(&key), + Nulls::Kept => true, + } +} + +fn environment(lookup: &dyn Lookup) -> impl Iterator + '_ { + keys::SETTINGS.iter().filter_map(|setting| { + let source = setting.environment_name()?; + let value = lookup.get(&source)?; + Some(( + setting.kwarg.to_owned(), + Entry { + value: Value::String(value), + source, + }, + )) + }) +} + +// load_config resolves os.environ/ to None when unset, so a missing reference is a null like any other +fn environment_reference(value: &Value, lookup: &dyn Lookup) -> Value { + match value { + Value::String(text) => match text.strip_prefix("os.environ/") { + Some(name) => lookup.get(name).map_or(Value::Null, Value::String), + None => value.clone(), + }, + _ => value.clone(), + } +} + +fn is_set(kwargs: &Kwargs, key: &str) -> bool { + kwargs.get(key).is_some_and(|entry| !entry.value.is_null()) +} diff --git a/litellm-rust/crates/config/src/redis/options.rs b/litellm-rust/crates/config/src/redis/options.rs new file mode 100644 index 00000000000..5de83584d8e --- /dev/null +++ b/litellm-rust/crates/config/src/redis/options.rs @@ -0,0 +1,245 @@ +use std::{collections::BTreeMap, time::Duration}; + +use litellm_core_utils::settings::Lookup; +use litellm_python_compat::{BigInt, number}; +use litellm_secret_value::SecretValue; + +use super::{ + Expected, RedisError, + keys::{self, Handling}, + layers::Entry, +}; +use crate::Value; + +pub(super) type Kwargs = BTreeMap; + +/// Every setting the resolver reads, after layering. A setting is `None` when it is unset, null or +/// empty, and then its environment fallback, if Python has one, stands in for it. +pub(super) struct Options { + pub(super) url: Option, + pub(super) host: Option, + pub(super) port: Option, + pub(super) db: Option, + pub(super) startup_nodes: Option, + pub(super) sentinel: SentinelOptions, + pub(super) username: Option, + pub(super) password: Option, + pub(super) credential_provider: Option, + pub(super) gcp: GcpOptions, + pub(super) azure: AzureOptions, + pub(super) aws: AwsOptions, + pub(super) ssl: Option, + pub(super) tls: TlsInput, + pub(super) namespace: Option, + pub(super) socket_timeout: Option, + pub(super) socket_connect_timeout: Option, +} + +pub(super) struct SentinelOptions { + pub(super) nodes: Option, + pub(super) service_name: Option, + pub(super) password: Option, +} + +pub(super) struct GcpOptions { + pub(super) service_account: Option, + pub(super) ssl_ca_certs: Option, +} + +pub(super) struct AzureOptions { + pub(super) ad_token: Option, + pub(super) client_id: Option, + pub(super) tenant_id: Option, + pub(super) client_secret: Option, +} + +pub(super) struct AwsOptions { + pub(super) auth: Option, + pub(super) user_name: Option, + pub(super) cache_name: Option, + pub(super) region: Option, + pub(super) serverless: Option, +} + +/// The TLS settings, read from the kwargs or from a `rediss://` url's query. +pub(super) struct TlsInput { + pub(super) cert_reqs: Option, + pub(super) ca_certs: Option, + pub(super) certfile: Option, + pub(super) keyfile: Option, + pub(super) check_hostname: Option, +} + +impl Options { + /// The typed settings and where each accepted-but-ignored one came from. + pub(super) fn read( + kwargs: Kwargs, + lookup: &dyn Lookup, + ) -> Result<(Self, Vec), RedisError> { + if let Some((_, entry)) = kwargs + .iter() + .find(|(key, _)| handling(key) == Some(Handling::Reject)) + { + return Err(RedisError::Unsupported { + setting: entry.source.clone(), + }); + } + let ignored = kwargs + .iter() + .filter(|(key, _)| handling(key) == Some(Handling::Warn)) + .map(|(_, entry)| entry.source.clone()) + .collect(); + let get = |key: &str| { + kwargs + .get(key) + .filter(|entry| is_present(&entry.value)) + .cloned() + .or_else(|| keys::fallback(key, lookup)) + }; + let options = Self { + url: get("url"), + host: get("host"), + port: get("port"), + db: get("db"), + startup_nodes: get("startup_nodes"), + sentinel: SentinelOptions { + nodes: get("sentinel_nodes"), + service_name: get("service_name"), + password: get("sentinel_password"), + }, + username: get("username"), + password: get("password"), + credential_provider: get("credential_provider"), + gcp: GcpOptions { + service_account: get("gcp_service_account"), + ssl_ca_certs: get("gcp_ssl_ca_certs"), + }, + azure: AzureOptions { + ad_token: get("azure_redis_ad_token"), + client_id: get("azure_client_id"), + tenant_id: get("azure_tenant_id"), + client_secret: get("azure_client_secret"), + }, + aws: AwsOptions { + auth: get("aws_iam_auth"), + user_name: get("aws_iam_user_name"), + cache_name: get("aws_iam_cache_name"), + region: get("aws_iam_region"), + serverless: get("aws_iam_serverless"), + }, + ssl: get("ssl"), + tls: TlsInput { + cert_reqs: get("ssl_cert_reqs"), + ca_certs: get("ssl_ca_certs"), + certfile: get("ssl_certfile"), + keyfile: get("ssl_keyfile"), + check_hostname: get("ssl_check_hostname"), + }, + namespace: get("namespace"), + socket_timeout: get("socket_timeout"), + socket_connect_timeout: get("socket_connect_timeout"), + }; + Ok((options, ignored)) + } +} + +fn handling(key: &str) -> Option { + keys::setting(key).map(|setting| setting.handling) +} + +// Python reads most of these with `x or fallback`, so an empty value counts as unset everywhere +fn is_present(value: &Value) -> bool { + match value { + Value::Null => false, + Value::String(text) => !text.is_empty(), + Value::Sequence(items) => !items.is_empty(), + Value::Mapping(entries) => !entries.is_empty(), + Value::Bool(_) | Value::Number(_) | Value::Tagged(_) => true, + } +} + +pub(super) fn invalid(setting: &str, expected: Expected) -> RedisError { + RedisError::Invalid { + setting: setting.to_owned(), + expected, + } +} + +pub(super) fn as_text(value: &Value) -> Option { + match value { + Value::String(text) => Some(text.clone()), + Value::Number(number) => Some(number.to_string()), + Value::Bool(flag) => Some(flag.to_string()), + _ => None, + } +} + +pub(super) fn text(entry: Option<&Entry>) -> Option { + entry.and_then(|entry| as_text(&entry.value)) +} + +pub(super) fn secret(entry: Option<&Entry>) -> Option { + text(entry).map(SecretValue::new) +} + +/// A flag every Python surface reads the same way. `_str_to_bool` takes only `true`, `1` and `yes` +/// as true while the proxy's pydantic models also take `on`, `t` and `y`, so text they disagree on, +/// or text neither accepts, is an error instead of a guess. +pub(super) fn flag(entry: Option<&Entry>) -> Result { + let Some(entry) = entry else { + return Ok(false); + }; + match &entry.value { + Value::Bool(flag) => Ok(*flag), + Value::Number(number) => Ok(number.as_f64().is_some_and(|number| number != 0.0)), + Value::String(text) => match text.to_ascii_lowercase().as_str() { + "true" | "1" | "yes" => Ok(true), + "false" | "0" | "no" | "off" | "f" | "n" => Ok(false), + _ => Err(invalid(&entry.source, Expected::Boolean)), + }, + _ => Err(invalid(&entry.source, Expected::Boolean)), + } +} + +pub(super) fn parsed( + entry: Option<&Entry>, + expected: Expected, + parse: impl Fn(&str) -> Option, +) -> Result, RedisError> { + entry + .and_then(|entry| Some((as_text(&entry.value)?, &entry.source))) + .map(|(text, source)| parse(&text).ok_or_else(|| invalid(source, expected))) + .transpose() +} + +pub(super) fn integer>(entry: Option<&Entry>) -> Result, RedisError> { + coerced(entry, Expected::Integer, number::int, |number| { + T::try_from(number).ok() + }) +} + +pub(super) fn seconds(entry: Option<&Entry>) -> Result, RedisError> { + coerced(entry, Expected::Seconds, number::float, |seconds| { + Duration::try_from_secs_f64(seconds).ok() + }) +} + +fn coerced( + entry: Option<&Entry>, + expected: Expected, + parse: impl Fn(&str) -> Option, + convert: impl Fn(N) -> Option, +) -> Result, RedisError> { + let Some((entry, text)) = entry.and_then(|entry| Some((entry, as_text(&entry.value)?))) else { + return Ok(None); + }; + match parse(&text) { + Some(number) => convert(number) + .map(Some) + .ok_or_else(|| invalid(&entry.source, expected)), + None => { + tracing::warn!(setting = %entry.source, "Redis setting is not a number, so the default is used like redis-py"); + Ok(None) + } + } +} diff --git a/litellm-rust/crates/config/src/redis/secrets.rs b/litellm-rust/crates/config/src/redis/secrets.rs new file mode 100644 index 00000000000..eba44e31d0f --- /dev/null +++ b/litellm-rust/crates/config/src/redis/secrets.rs @@ -0,0 +1,62 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use litellm_core_utils::settings::Lookup; +use litellm_secret_value::SecretValue; + +use super::{ConfiguredRedis, Value, keys::SETTINGS}; + +#[derive(Clone, Debug, Default)] +pub struct SecretSnapshot(BTreeMap>); + +impl SecretSnapshot { + pub fn new(values: BTreeMap>) -> Self { + Self(values) + } + + pub fn read(names: BTreeSet, lookup: &dyn Lookup) -> Self { + Self( + names + .into_iter() + .map(|name| { + let value = lookup.get(&name).map(SecretValue::new); + (name, value) + }) + .collect(), + ) + } +} + +impl Lookup for SecretSnapshot { + fn get(&self, name: &str) -> Option { + match self.0.get(name) { + Some(value) => value.as_ref().map(|value| value.expose().to_owned()), + None => { + debug_assert!( + false, + "{name} is missing from ConfiguredRedis::secret_names" + ); + tracing::error!( + name, + "a Redis secret was read without being in the snapshot" + ); + None + } + } + } +} + +impl ConfiguredRedis { + pub fn secret_names(&self) -> BTreeSet { + let references = self.fields.values().filter_map(|value| match value { + Value::String(text) => text.strip_prefix("os.environ/").map(str::to_owned), + _ => None, + }); + let declared = SETTINGS.iter().flat_map(|setting| { + setting + .environment_name() + .into_iter() + .chain(setting.fallback.iter().map(|name| (*name).to_owned())) + }); + declared.chain(references).collect() + } +} diff --git a/litellm-rust/crates/config/src/redis/tls.rs b/litellm-rust/crates/config/src/redis/tls.rs new file mode 100644 index 00000000000..1157e01832e --- /dev/null +++ b/litellm-rust/crates/config/src/redis/tls.rs @@ -0,0 +1,91 @@ +use litellm_redis_types::TlsOptions; + +use super::{ + Expected, RedisError, + layers::Entry, + options::{TlsInput, flag, parsed, text}, +}; +use crate::Value; + +// redis-py's parse_url turns every query parameter into a client kwarg. Only the TLS ones have a redis-rs +// equivalent, so any other parameter, and any parameter on a plain redis:// url, is rejected +const URL_TLS_SETTINGS: &[&str] = &[ + "ssl_cert_reqs", + "ssl_ca_certs", + "ssl_certfile", + "ssl_keyfile", + "ssl_check_hostname", +]; + +/// `gcp_ca_certs` replaces `ssl_ca_certs` when GCP IAM is on, as `_get_redis_client_logic` does. +pub(super) fn tls_options( + input: &TlsInput, + gcp_ca_certs: Option<&Entry>, +) -> Result { + Ok(TlsOptions { + ca_certs: text(gcp_ca_certs.or(input.ca_certs.as_ref())).map(Into::into), + certfile: text(input.certfile.as_ref()).map(Into::into), + keyfile: text(input.keyfile.as_ref()).map(Into::into), + verify_certificate: verify_certificate(input.cert_reqs.as_ref())?, + check_hostname: flag(input.check_hostname.as_ref())?, + }) +} + +fn verify_certificate(cert_reqs: Option<&Entry>) -> Result { + parsed(cert_reqs, Expected::CertRequirement, |text| match text { + "required" | "optional" => Some(true), + "none" => Some(false), + _ => None, + }) + .map(|verify| verify.unwrap_or(true)) +} + +pub(super) fn url_tls(url: &str) -> Result { + let Ok(parsed) = url::Url::parse(url) else { + return Ok(TlsOptions::verified()); + }; + let tls_scheme = parsed.scheme().eq_ignore_ascii_case("rediss"); + let query: Vec<(String, Entry)> = parsed + .query_pairs() + .map(|(name, value)| { + ( + name.to_string(), + Entry { + value: Value::String(value.into_owned()), + source: format!("url ?{name}"), + }, + ) + }) + .collect(); + if let Some((_, entry)) = query + .iter() + .find(|(name, _)| !tls_scheme || !URL_TLS_SETTINGS.contains(&name.as_str())) + { + return Err(RedisError::Unsupported { + setting: entry.source.clone(), + }); + } + let get = |key: &str| { + query + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, entry)| entry.clone()) + }; + Ok(TlsOptions { + ca_certs: text(get("ssl_ca_certs").as_ref()).map(Into::into), + certfile: text(get("ssl_certfile").as_ref()).map(Into::into), + keyfile: text(get("ssl_keyfile").as_ref()).map(Into::into), + verify_certificate: verify_certificate(get("ssl_cert_reqs").as_ref())?, + check_hostname: text(get("ssl_check_hostname").as_ref()) + .is_some_and(|text| url_bool(&text)), + }) +} + +// redis-py's to_bool: only 0, F, FALSE, N and NO (any case) are false +fn url_bool(text: &str) -> bool { + !matches!( + text.to_ascii_uppercase().as_str(), + "" | "0" | "F" | "FALSE" | "N" | "NO" + ) +} diff --git a/litellm-rust/crates/config/tests/redis.rs b/litellm-rust/crates/config/tests/redis.rs index ab409541c8e..93db5b4de70 100644 --- a/litellm-rust/crates/config/tests/redis.rs +++ b/litellm-rust/crates/config/tests/redis.rs @@ -1,42 +1,86 @@ -use std::{ - collections::HashMap, - fmt::Write, - sync::{Arc, Mutex}, +use std::collections::{BTreeMap, HashMap}; + +use litellm_config::{ + CacheSettings, Config, ConfiguredRedis, Expected, RedisError, Resolution, SecretSnapshot, Value, +}; +use litellm_redis_types::{ + AwsIamAuth, AzureAdAuth, ClusterNode, Endpoint, RedisAuth, ResolvedRedis, TlsOptions, }; - -use litellm_config::{CacheSettings, Config, Error, RedisEndpoint, RedisSettings, ResolvedRedis}; use rstest::rstest; -use tracing::{Event, Subscriber, field::Visit}; -use tracing_subscriber::{Layer, layer::Context, prelude::*}; +use serde_json::json; -fn resolve( +fn resolution( + configured: &ConfiguredRedis, + environment: &HashMap, +) -> Result { + let lookup = |name: &str| environment.get(name).cloned(); + configured.resolve( + &SecretSnapshot::read(configured.secret_names(), &lookup), + &lookup, + ) +} + +fn redis_cache_kwargs(kwargs: serde_json::Value) -> ConfiguredRedis { + ConfiguredRedis::redis_cache_kwargs( + serde_json::from_value::>(kwargs).unwrap(), + ) +} + +fn resolve_kwargs( + kwargs: serde_json::Value, + environment: &HashMap, +) -> Result { + resolution(&redis_cache_kwargs(kwargs), environment).map(|resolution| resolution.redis) +} + +fn coordination_resolution( yaml: &str, environment: &[(&str, &str)], -) -> Result { +) -> Result { let environment: HashMap = environment .iter() .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) .collect(); let config = Config::from_yaml(yaml).unwrap(); - config + let configured = config .general_settings .coordination_redis - .as_ref() - .unwrap_or(&RedisSettings::default()) - .resolve(&|name| environment.get(name).cloned()) + .unwrap_or_default() + .coordination()?; + resolution(&configured, &environment) } -fn resolve_cache(yaml: &str) -> Result { +fn resolve(yaml: &str, environment: &[(&str, &str)]) -> Result { + coordination_resolution(yaml, environment).map(|resolution| resolution.redis) +} + +fn resolve_cache(yaml: &str) -> Result { + resolve_cache_in(yaml, &[]) +} + +fn resolve_cache_in(yaml: &str, environment: &[(&str, &str)]) -> Result { + let lookup = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + }; let config = Config::from_yaml(yaml).unwrap(); - match config.litellm_settings.cache_params.unwrap() { - CacheSettings::Redis(settings) => settings.resolve_cache(&|_| None), + let configured = match config.litellm_settings.cache_params.unwrap() { + CacheSettings::Redis(settings) => settings.cache_params(), CacheSettings::Other(_) => panic!("expected a Redis cache"), - } + }; + configured + .resolve( + &SecretSnapshot::read(configured.secret_names(), &lookup), + &lookup, + ) + .map(|resolution| resolution.redis) } fn cluster_ports(resolved: &ResolvedRedis) -> Vec { match &resolved.endpoint { - RedisEndpoint::Cluster(cluster) => cluster.nodes.iter().map(|node| node.port).collect(), + Endpoint::Cluster(cluster) => cluster.nodes.iter().map(|node| node.port).collect(), endpoint => panic!("expected a cluster, got {endpoint:?}"), } } @@ -47,46 +91,6 @@ fn cluster_ports(resolved: &ResolvedRedis) -> Vec { &[], "host:redis:6380:2:true:app:1500" )] -#[case::environment_url( - "model_list: []", - &[("REDIS_URL", "redis://environment:6379")], - "url:redis://environment:6379" -)] -#[case::explicit_host_drops_environment_url( - "general_settings:\n coordination_redis:\n host: configured", - &[("REDIS_URL", "redis://environment:6379")], - "host:configured:6379:0:false::none" -)] -#[case::yaml_cluster( - "general_settings:\n coordination_redis:\n startup_nodes:\n - host: node-a\n port: 7000\n - host: node-b\n port: \"7001\"", - &[], - "cluster:node-a:7000,node-b:7001" -)] -#[case::json_environment_cluster( - "model_list: []", - &[("REDIS_CLUSTER_NODES", r#"[{"host":"node-a","port":"7000"},{"host":"node-b","port":7001}]"#)], - "cluster:node-a:7000,node-b:7001" -)] -#[case::missing_reference_keeps_environment_fallback( - "general_settings:\n coordination_redis:\n host: os.environ/MISSING_HOST", - &[("REDIS_HOST", "environment")], - "host:environment:6379:0:false::none" -)] -#[case::configured_values_beat_the_same_environment_values( - "general_settings:\n coordination_redis:\n host: configured\n port: 6380\n db: 3", - &[("REDIS_HOST", "environment"), ("REDIS_PORT", "6390"), ("REDIS_DB", "9")], - "host:configured:6380:3:false::none" -)] -#[case::environment_url_beats_the_environment_host( - "model_list: []", - &[("REDIS_URL", "redis://environment:6379"), ("REDIS_HOST", "ignored")], - "url:redis://environment:6379" -)] -#[case::startup_nodes_beat_a_url( - "general_settings:\n coordination_redis:\n url: redis://single:6379\n startup_nodes:\n - host: node-a\n port: 7000", - &[], - "cluster:node-a:7000" -)] #[case::environment_references_resolve_the_host_and_port( "general_settings:\n coordination_redis:\n host: os.environ/COORD_HOST\n port: os.environ/COORD_PORT", &[("COORD_HOST", "referenced"), ("COORD_PORT", "6400")], @@ -104,20 +108,21 @@ fn resolves_redis_through_the_parsed_config( ) { let resolved = resolve(yaml, environment).unwrap(); let description = match resolved.endpoint { - RedisEndpoint::Url(url) => format!("url:{}", url.expose()), - RedisEndpoint::Host(host) => format!( + Endpoint::Url { url, .. } => format!("url:{}", url.expose()), + Endpoint::Sentinel(sentinel) => format!("sentinel:{}", sentinel.service_name), + Endpoint::Host(host) => format!( "host:{}:{}:{}:{}:{}:{}", host.host, host.port, host.db, - host.tls, + host.tls.is_some(), resolved.namespace.unwrap_or_default(), resolved .timeout .map(|timeout| timeout.as_millis().to_string()) .unwrap_or_else(|| "none".to_owned()) ), - RedisEndpoint::Cluster(cluster) => format!( + Endpoint::Cluster(cluster) => format!( "cluster:{}", cluster .nodes @@ -132,50 +137,20 @@ fn resolves_redis_through_the_parsed_config( } #[rstest] -#[case::custom_tls_file( - "general_settings:\n coordination_redis:\n host: redis\n ssl_ca_certs: /ca.pem", +#[case::tls_ciphers( + "general_settings:\n coordination_redis:\n host: redis\n ssl_ciphers: HIGH", &[], - "ssl_ca_certs" + "ssl_ciphers" )] -#[case::gcp_auth_from_config( - "general_settings:\n coordination_redis:\n host: redis\n gcp_service_account: account", - &[], - "gcp_service_account" -)] -#[case::gcp_auth_from_environment( +#[case::ssl_ciphers_from_environment( "model_list: []", - &[("REDIS_HOST", "redis"), ("REDIS_GCP_SERVICE_ACCOUNT", "account")], - "REDIS_GCP_SERVICE_ACCOUNT" + &[("REDIS_HOST", "redis"), ("REDIS_SSL_CIPHERS", "HIGH")], + "REDIS_SSL_CIPHERS" )] -#[case::azure_auth_from_config( - "general_settings:\n coordination_redis:\n host: redis\n azure_redis_ad_token: true", +#[case::connection_pool( + "general_settings:\n coordination_redis:\n host: redis\n connection_pool: shared", &[], - "azure_redis_ad_token" -)] -#[case::azure_auth_from_environment( - "model_list: []", - &[("REDIS_HOST", "redis"), ("REDIS_AZURE_AD_TOKEN", "true")], - "REDIS_AZURE_AD_TOKEN" -)] -#[case::aws_auth_from_config( - "general_settings:\n coordination_redis:\n host: redis\n aws_iam_auth: true", - &[], - "aws_iam_auth" -)] -#[case::aws_auth_from_environment( - "model_list: []", - &[("REDIS_HOST", "redis"), ("REDIS_AWS_IAM_AUTH", "true")], - "REDIS_AWS_IAM_AUTH" -)] -#[case::sentinel_from_config( - "general_settings:\n coordination_redis:\n sentinel_nodes: [[redis, 26379]]", - &[], - "sentinel_nodes" -)] -#[case::sentinel_from_environment( - "model_list: []", - &[("REDIS_SENTINEL_NODES", "[[\"redis\",26379]]")], - "REDIS_SENTINEL_NODES" + "connection_pool" )] fn rejected_settings_name_their_source( #[case] yaml: &str, @@ -184,7 +159,14 @@ fn rejected_settings_name_their_source( ) { let error = resolve(yaml, environment).unwrap_err(); - assert!(matches!(error, Error::RedisSetting(setting) if setting == expected)); + assert!(matches!(error, RedisError::Unsupported { setting } if setting == expected)); +} + +fn static_auth(username: Option<&str>, password: Option<&str>) -> RedisAuth { + RedisAuth::Static { + username: username.map(litellm_secret_value::SecretValue::new), + password: password.map(litellm_secret_value::SecretValue::new), + } } #[rstest] @@ -198,14 +180,308 @@ fn an_explicit_host_keeps_environment_credentials() { ) .unwrap(); - let RedisEndpoint::Host(host) = resolved.endpoint else { - panic!("expected a host endpoint"); + assert_eq!(resolved.auth, static_auth(None, Some("environment-secret"))); + assert!(matches!(resolved.endpoint, Endpoint::Host(host) if host.tls.is_some())); +} + +fn azure(username: Option<&str>) -> RedisAuth { + RedisAuth::AzureAd(AzureAdAuth { + client_id: Some("client".to_owned()), + tenant_id: None, + client_secret: None, + username: username.map(str::to_owned), + }) +} + +fn aws(serverless: bool) -> RedisAuth { + RedisAuth::AwsIam(AwsIamAuth { + user_name: "iam-user".to_owned(), + cache_name: "cache-1".to_owned(), + region: "us-east-1".to_owned(), + serverless, + }) +} + +#[rstest] +#[case::static_credentials( + "general_settings:\n coordination_redis:\n host: redis\n username: user\n password: pass", + &[], + static_auth(Some("user"), Some("pass")) +)] +#[case::an_explicit_provider_beats_every_cloud_auth( + "general_settings:\n coordination_redis:\n host: redis\n password: pass\n credential_provider: host-object\n gcp_service_account: account\n aws_iam_auth: true", + &[("REDIS_AZURE_AD_TOKEN", "true")], + RedisAuth::Host +)] +#[case::gcp_beats_azure_and_aws( + "general_settings:\n coordination_redis:\n host: redis\n password: pass\n azure_redis_ad_token: \"True\"\n aws_iam_auth: true", + &[("REDIS_GCP_SERVICE_ACCOUNT", "projects/-/serviceAccounts/sa@p.iam.gserviceaccount.com")], + RedisAuth::GcpIam { service_account: "projects/-/serviceAccounts/sa@p.iam.gserviceaccount.com".to_owned() } +)] +#[case::azure_beats_aws_and_takes_the_environment_username( + "general_settings:\n coordination_redis:\n host: redis\n username: configured-user\n azure_redis_ad_token: true\n azure_client_id: client\n aws_iam_auth: true", + &[("REDIS_USERNAME", "environment-user")], + azure(Some("environment-user")) +)] +#[case::azure_without_an_environment_username_sends_only_the_token( + "general_settings:\n coordination_redis:\n host: redis\n username: configured-user\n azure_redis_ad_token: true\n azure_client_id: client", + &[], + azure(None) +)] +#[case::azure_needs_the_literal_true( + "general_settings:\n coordination_redis:\n host: redis\n azure_redis_ad_token: \"1\"", + &[], + static_auth(None, None) +)] +#[case::aws_with_a_region_from_the_environment( + "general_settings:\n coordination_redis:\n host: redis\n ssl: true\n password: dropped\n aws_iam_auth: \"yes\"\n aws_iam_user_name: iam-user\n aws_iam_cache_name: cache-1", + &[("AWS_DEFAULT_REGION", "us-east-1")], + aws(false) +)] +#[case::aws_serverless_over_a_rediss_url( + "general_settings:\n coordination_redis:\n url: rediss://cache-1:6379\n aws_iam_auth: true\n aws_iam_user_name: iam-user\n aws_iam_cache_name: cache-1\n aws_iam_region: us-east-1\n aws_iam_serverless: true", + &[], + aws(true) +)] +fn auth_follows_python_precedence( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: RedisAuth, +) { + assert_eq!(resolve(yaml, environment).unwrap().auth, expected); +} + +#[rstest] +#[case::without_tls( + "general_settings:\n coordination_redis:\n host: redis\n aws_iam_auth: true\n aws_iam_user_name: u\n aws_iam_cache_name: c\n aws_iam_region: r", + "requires TLS" +)] +#[case::a_plain_url_ignores_ssl( + "general_settings:\n coordination_redis:\n url: redis://cache:6379\n ssl: true\n aws_iam_auth: true\n aws_iam_user_name: u\n aws_iam_cache_name: c\n aws_iam_region: r", + "requires TLS" +)] +#[case::missing_settings( + "general_settings:\n coordination_redis:\n host: redis\n ssl: true\n aws_iam_auth: true\n aws_iam_cache_name: c", + "requires: aws_iam_user_name, aws_iam_region" +)] +fn incomplete_aws_iam_auth_fails(#[case] yaml: &str, #[case] message: &str) { + let error = resolve(yaml, &[]).unwrap_err(); + + assert!(error.to_string().contains(message), "{error}"); +} + +#[rstest] +#[case::defaults_verify_the_certificate_but_not_the_hostname( + " ssl: true", + &[], + Some(TlsOptions { ca_certs: None, certfile: None, keyfile: None, verify_certificate: true, check_hostname: false }) +)] +#[case::every_option( + " ssl: true\n ssl_ca_certs: /ca.pem\n ssl_certfile: /client.pem\n ssl_keyfile: /client.key\n ssl_cert_reqs: none\n ssl_check_hostname: true", + &[], + Some(TlsOptions { ca_certs: Some("/ca.pem".into()), certfile: Some("/client.pem".into()), keyfile: Some("/client.key".into()), verify_certificate: false, check_hostname: true }) +)] +#[case::options_from_the_environment( + " ssl: true", + &[("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional")], + Some(TlsOptions { ca_certs: Some("/env-ca.pem".into()), certfile: None, keyfile: None, verify_certificate: true, check_hostname: false }) +)] +#[case::gcp_ca_certs_replace_the_configured_ones( + " ssl: true\n ssl_ca_certs: /ca.pem\n gcp_service_account: account\n gcp_ssl_ca_certs: /gcp-ca.pem", + &[], + Some(TlsOptions { ca_certs: Some("/gcp-ca.pem".into()), certfile: None, keyfile: None, verify_certificate: true, check_hostname: false }) +)] +#[case::options_without_ssl_stay_plaintext(" ssl_ca_certs: /ca.pem", &[], None)] +fn tls_options_resolve_like_redis_py( + #[case] block: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: Option, +) { + let yaml = format!("general_settings:\n coordination_redis:\n host: redis\n{block}"); + + let resolved = resolve(&yaml, environment).unwrap(); + + assert!(matches!(resolved.endpoint, Endpoint::Host(host) if host.tls == expected)); +} + +#[rstest] +fn an_unknown_cert_requirement_fails() { + let error = resolve( + "general_settings:\n coordination_redis:\n host: redis\n ssl: true\n ssl_cert_reqs: strict", + &[], + ) + .unwrap_err(); + + assert!(matches!( + error, + RedisError::Invalid { setting, expected: Expected::CertRequirement } if setting == "ssl_cert_reqs" + )); +} + +#[rstest] +#[case::from_config( + "general_settings:\n coordination_redis:\n sentinel_nodes: [[sentinel-a, 26379], [sentinel-b, \"26380\"]]\n service_name: primary\n sentinel_password: sentinel-secret\n db: 2", + &[] +)] +#[case::from_the_environment( + "model_list: []", + &[ + ("REDIS_SENTINEL_NODES", r#"[["sentinel-a", 26379], ["sentinel-b", "26380"]]"#), + ("REDIS_SERVICE_NAME", "primary"), + ("REDIS_SENTINEL_PASSWORD", "sentinel-secret"), + ("REDIS_DB", "2"), + ] +)] +fn sentinel_nodes_resolve_to_a_sentinel_endpoint( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], +) { + let Endpoint::Sentinel(sentinel) = resolve(yaml, environment).unwrap().endpoint else { + panic!("expected a sentinel endpoint"); }; + assert_eq!( - host.password.as_ref().map(|password| password.expose()), - Some("environment-secret") + sentinel.sentinels, + vec![ + ClusterNode { + host: "sentinel-a".to_owned(), + port: 26379 + }, + ClusterNode { + host: "sentinel-b".to_owned(), + port: 26380 + }, + ] + ); + assert_eq!(sentinel.service_name, "primary"); + assert_eq!( + sentinel + .sentinel_password + .as_ref() + .map(|password| password.expose()), + Some("sentinel-secret") + ); + assert_eq!(sentinel.db, 2); +} + +#[rstest] +#[case::a_url_beats_sentinels( + "general_settings:\n coordination_redis:\n url: redis://single:6379\n sentinel_nodes: [[s, 26379]]\n service_name: primary", + "url:redis://single:6379" +)] +fn sentinel_precedence(#[case] yaml: &str, #[case] expected: &str) { + let resolved = resolve(yaml, &[]).unwrap(); + let description = match resolved.endpoint { + Endpoint::Url { url, .. } => format!("url:{}", url.expose()), + Endpoint::Sentinel(sentinel) => format!("sentinel:{}", sentinel.service_name), + other => panic!("unexpected {other:?}"), + }; + + assert_eq!(description, expected); +} + +#[rstest] +fn sentinels_without_a_service_name_fail() { + let error = resolve( + "general_settings:\n coordination_redis:\n sentinel_nodes: [[s, 26379]]", + &[], + ) + .unwrap_err(); + + assert!(matches!(error, RedisError::NoSentinelService), "{error:?}"); +} + +#[rstest] +#[case::configured_host( + "general_settings:\n coordination_redis:\n host: ignored\n sentinel_nodes: [[s, 26379]]\n service_name: primary", + &[], + "host" +)] +#[case::environment_port( + "general_settings:\n coordination_redis:\n sentinel_nodes: [[s, 26379]]\n service_name: primary", + &[("REDIS_PORT", "6380")], + "REDIS_PORT" +)] +fn a_host_or_port_cannot_reach_sentinel( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!(&error, RedisError::SentinelWithHost { setting } if setting == source), + "{error:?}" + ); +} + +#[rstest] +#[case::not_pairs( + "general_settings:\n coordination_redis:\n sentinel_nodes: [{host: s, port: 26379}]\n service_name: primary", + &[], + "sentinel_nodes" +)] +#[case::from_the_environment( + "model_list: []", + &[("REDIS_SENTINEL_NODES", "not-json"), ("REDIS_SERVICE_NAME", "primary")], + "REDIS_SENTINEL_NODES" +)] +fn invalid_sentinel_nodes_name_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!(&error, RedisError::Invalid { setting, expected: Expected::SentinelNodes } if setting == source), + "{error:?}" + ); +} + +#[rstest] +#[case::from_config("general_settings:\n coordination_redis:\n startup_nodes: [{host: a}]", &[], "startup_nodes")] +#[case::from_the_environment("model_list: []", &[("REDIS_CLUSTER_NODES", "not-json")], "REDIS_CLUSTER_NODES")] +#[case::out_of_range_port_from_the_environment("model_list: []", &[("REDIS_HOST", "redis"), ("REDIS_PORT", "70000")], "REDIS_PORT")] +fn invalid_values_name_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!(&error, RedisError::Invalid { setting, .. } if setting == source), + "{error:?}" + ); +} + +#[rstest] +#[case::falls_back_to_the_socket_timeout(" socket_timeout: 1.5", Some(1500), None)] +#[case::explicit_connect_timeout( + " socket_timeout: 1.5\n socket_connect_timeout: 0.25", + Some(1500), + Some(250) +)] +fn connect_timeout_is_resolved_separately( + #[case] block: &str, + #[case] timeout_ms: Option, + #[case] connect_ms: Option, +) { + let resolved = resolve( + &format!("general_settings:\n coordination_redis:\n host: redis\n{block}"), + &[], + ) + .unwrap(); + + assert_eq!( + resolved.timeout.map(|timeout| timeout.as_millis()), + timeout_ms + ); + assert_eq!( + resolved.connect_timeout.map(|timeout| timeout.as_millis()), + connect_ms ); - assert!(host.tls); } #[rstest] @@ -217,13 +493,7 @@ fn an_explicit_host_keeps_environment_credentials() { fn a_block_without_a_usable_target_fails(#[case] yaml: &str, #[case] environment: &[(&str, &str)]) { let error = resolve(yaml, environment).unwrap_err(); - assert!(matches!( - error, - Error::RedisInvalid { - setting: "host", - .. - } - )); + assert!(matches!(error, RedisError::NoTarget), "{error:?}"); } #[rstest] @@ -253,7 +523,7 @@ fn a_null_canonical_startup_nodes_setting_shadows_the_cache_alias() { ) .unwrap(); - assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); + assert!(matches!(resolved.endpoint, Endpoint::Host(_))); } #[rstest] @@ -263,7 +533,7 @@ fn a_missing_canonical_startup_nodes_reference_shadows_the_cache_alias() { ) .unwrap(); - assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); + assert!(matches!(resolved.endpoint, Endpoint::Host(_))); } #[rstest] @@ -272,19 +542,19 @@ fn rejected_environment_settings_never_print_the_value() { "model_list: []", &[ ("REDIS_HOST", "redis"), - ("REDIS_SSL_CERT_REQS", "private-option-value"), + ("REDIS_SSL_CIPHERS", "private-option-value"), ], ) .unwrap_err(); let message = error.to_string(); - assert!(message.contains("REDIS_SSL_CERT_REQS")); + assert!(message.contains("REDIS_SSL_CIPHERS")); assert!(!message.contains("private-option-value")); } #[rstest] #[case::negative("-1")] -#[case::not_a_number("slow")] +#[case::infinite("inf")] fn rejects_bad_socket_timeout(#[case] timeout: &str) { let yaml = format!( "general_settings:\n coordination_redis:\n host: redis\n socket_timeout: {timeout}" @@ -293,24 +563,10 @@ fn rejects_bad_socket_timeout(#[case] timeout: &str) { assert!(matches!( error, - Error::RedisInvalid { - setting: "socket_timeout", - .. - } + RedisError::Invalid { setting, expected: Expected::Seconds } if setting == "socket_timeout" )); } -#[rstest] -fn defaults_socket_timeout_for_the_gateway() { - let resolved = resolve( - "general_settings:\n coordination_redis:\n host: redis", - &[], - ) - .unwrap(); - - assert_eq!(resolved.timeout, None); -} - #[rstest] fn debug_redacts_redis_secrets_and_url() { let config = Config::from_yaml( @@ -318,8 +574,9 @@ fn debug_redacts_redis_secrets_and_url() { ) .unwrap(); let settings = config.general_settings.coordination_redis.unwrap(); - let resolved = settings.resolve(&|_| None).unwrap(); - let debug = format!("{settings:?} {resolved:?}"); + let configured = settings.coordination().unwrap(); + let resolved = resolution(&configured, &HashMap::new()).unwrap().redis; + let debug = format!("{settings:?} {configured:?} {resolved:?}"); assert!(!debug.contains("redis://user:password")); assert!(!debug.contains("credential-user")); @@ -361,45 +618,538 @@ fn legacy_unknown_settings_do_not_change_resolution() { ) .unwrap(); - assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); -} - -struct WarningCapture(Arc>>); - -impl Layer for WarningCapture { - fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) { - let mut fields = Fields::default(); - event.record(&mut fields); - self.0.lock().unwrap().push(fields.0); - } -} - -#[derive(Default)] -struct Fields(String); - -impl Visit for Fields { - fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { - write!(self.0, "{}={value:?} ", field.name()).unwrap(); - } - - fn record_str(&mut self, field: &tracing::field::Field, value: &str) { - write!(self.0, "{}={value} ", field.name()).unwrap(); - } + assert!(matches!(resolved.endpoint, Endpoint::Host(_))); } #[rstest] -fn ignored_setting_warning_names_the_setting_without_its_value() { - let events = Arc::new(Mutex::new(Vec::new())); - let subscriber = tracing_subscriber::registry().with(WarningCapture(events.clone())); - tracing::subscriber::with_default(subscriber, || { - resolve( - "general_settings:\n coordination_redis:\n host: redis\n max_connections: 47", - &[], - ) - .unwrap(); - }); - let warnings = events.lock().unwrap().join(" "); +#[case::configured( + "general_settings:\n coordination_redis:\n host: redis\n max_connections: 47", + &[], + "max_connections" +)] +#[case::from_the_environment( + "general_settings:\n coordination_redis:\n host: redis", + &[("REDIS_MAX_CONNECTIONS", "47")], + "REDIS_MAX_CONNECTIONS" +)] +fn ignored_settings_are_named_by_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let resolution = coordination_resolution(yaml, environment).unwrap(); - assert!(warnings.contains("max_connections")); - assert!(!warnings.contains("47")); + assert_eq!(resolution.ignored, vec![source.to_owned()]); +} + +fn tls_view(tls: Option<&TlsOptions>) -> serde_json::Value { + tls.map_or(serde_json::Value::Null, |tls| { + serde_json::json!({ + "ca_certs": tls.ca_certs, + "certfile": tls.certfile, + "keyfile": tls.keyfile, + "verify_certificate": tls.verify_certificate, + "check_hostname": tls.check_hostname, + }) + }) +} + +fn nodes_view(nodes: &[ClusterNode]) -> Vec { + nodes + .iter() + .map(|node| format!("{}:{}", node.host, node.port)) + .collect() +} + +fn expose(value: Option<&litellm_secret_value::SecretValue>) -> Option<&str> { + value.map(litellm_secret_value::SecretValue::expose) +} + +fn expected_view(fields: serde_json::Value) -> serde_json::Value { + let serde_json::Value::Object(fields) = fields else { + unreachable!("expected views are objects") + }; + let defaults = json!({ + "auth": {"kind": "static", "username": null, "password": null}, + "tls": null, + "socket_timeout": 5.0, + "socket_connect_timeout": 5.0, + }); + let serde_json::Value::Object(defaults) = defaults else { + unreachable!("the defaults are an object") + }; + serde_json::Value::Object(defaults.into_iter().chain(fields).collect()) +} + +fn view(resolved: &ResolvedRedis) -> serde_json::Value { + let target = match &resolved.endpoint { + Endpoint::Host(host) => serde_json::json!({ + "mode": "host", + "host": host.host, + "port": host.port, + "db": host.db, + "tls": tls_view(host.tls.as_ref()), + }), + Endpoint::Url { url, tls } => serde_json::json!({ + "mode": "url", + "url": url.expose(), + "tls": tls_view(url.expose().starts_with("rediss://").then_some(tls)), + }), + Endpoint::Cluster(cluster) => serde_json::json!({ + "mode": "cluster", + "nodes": nodes_view(&cluster.nodes), + "tls": tls_view(cluster.tls.as_ref()), + }), + Endpoint::Sentinel(sentinel) => serde_json::json!({ + "mode": "sentinel", + "nodes": nodes_view(&sentinel.sentinels), + "service_name": sentinel.service_name, + "sentinel_password": expose(sentinel.sentinel_password.as_ref()), + "db": sentinel.db, + "tls": tls_view(sentinel.tls.as_ref()), + }), + }; + let auth = match &resolved.auth { + RedisAuth::Static { username, password } => serde_json::json!({ + "kind": "static", + "username": expose(username.as_ref()), + "password": expose(password.as_ref()), + }), + RedisAuth::Host => serde_json::json!({"kind": "host"}), + RedisAuth::GcpIam { service_account } => { + serde_json::json!({"kind": "gcp_iam", "service_account": service_account}) + } + RedisAuth::AzureAd(azure) => { + serde_json::json!({"kind": "azure_ad", "username": azure.username}) + } + RedisAuth::AwsIam(aws) => serde_json::json!({ + "kind": "aws_iam", + "user_name": aws.user_name, + "cache_name": aws.cache_name, + "region": aws.region, + "serverless": aws.serverless, + }), + }; + let serde_json::Value::Object(target) = target else { + unreachable!("every target view is an object") + }; + let rest = serde_json::json!({ + "auth": auth, + "socket_timeout": resolved.timeout.map_or(5.0, |timeout| timeout.as_secs_f64()), + "socket_connect_timeout": resolved.connect_timeout.or(resolved.timeout).map_or(5.0, |timeout| timeout.as_secs_f64()), + }); + let serde_json::Value::Object(rest) = rest else { + unreachable!("the view is an object") + }; + serde_json::Value::Object(target.into_iter().chain(rest).collect()) +} + +#[rstest] +#[case::host_defaults( + json!({"host": "redis"}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::empty_cluster_nodes_use_the_environment( + json!({"startup_nodes": []}), + &[("REDIS_CLUSTER_NODES", r#"[{"host": "node-a", "port": 7000}]"#)], + Ok(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"]}))), +)] +#[case::empty_sentinel_nodes_use_the_environment( + json!({"sentinel_nodes": [], "service_name": "primary"}), + &[("REDIS_SENTINEL_NODES", r#"[["sentinel-a", 26379]]"#)], + Ok(expected_view(json!({"db": 0, "mode": "sentinel", "nodes": ["sentinel-a:26379"], "sentinel_password": null, "service_name": "primary"}))), +)] +#[case::empty_service_name_uses_the_environment( + json!({"sentinel_nodes": [["sentinel-a", 26379]], "service_name": ""}), + &[("REDIS_SERVICE_NAME", "primary")], + Ok(expected_view(json!({"db": 0, "mode": "sentinel", "nodes": ["sentinel-a:26379"], "sentinel_password": null, "service_name": "primary"}))), +)] +#[case::plaintext_ignores_tls_requirements( + json!({"host": "redis", "ssl_cert_reqs": "strict"}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::an_ambiguous_flag_is_refused( + json!({"host": "redis", "ssl": "on"}), + &[], + Err("invalid Redis setting ssl: expected a boolean"), +)] +#[case::empty_sentinel_nodes_without_the_environment_name_no_target( + json!({"sentinel_nodes": [], "service_name": "primary"}), + &[], + Err("either host or url must be specified for redis"), +)] +#[case::ssl_ca_path_is_unsupported( + json!({"host": "redis", "ssl": true, "ssl_ca_path": "/ca"}), + &[], + Err("Redis host does not yet support configured setting ssl_ca_path"), +)] +#[case::host_with_every_basic_setting( + json!({"db": 2, "host": "redis", "password": "pass", "port": 6380, "socket_timeout": 1.5, "ssl": true, "username": "user"}), + &[], + Ok(expected_view(json!({"auth": {"kind": "static", "password": "pass", "username": "user"}, "db": 2, "host": "redis", "mode": "host", "port": 6380, "socket_connect_timeout": 1.5, "socket_timeout": 1.5, "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}}))), +)] +#[case::environment_host_port_and_password( + json!({}), + &[("REDIS_HOST", "environment"), ("REDIS_PASSWORD", "environment-secret"), ("REDIS_PORT", "6390"), ("REDIS_SSL", "True")], + Ok(expected_view(json!({"auth": {"kind": "static", "password": "environment-secret", "username": null}, "db": 0, "host": "environment", "mode": "host", "port": 6390, "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}}))), +)] +#[case::string_numbers_are_coerced( + json!({"db": "3", "host": "redis", "port": "6380", "socket_timeout": "2.5"}), + &[], + Ok(expected_view(json!({"db": 3, "host": "redis", "mode": "host", "port": 6380, "socket_connect_timeout": 2.5, "socket_timeout": 2.5}))), +)] +#[case::padded_and_underscored_numbers_parse_like_python_int_and_float( + json!({"host": "redis", "port": "6380\n", "db": " 2 ", "socket_timeout": "1_0.5\n"}), + &[], + Ok(expected_view(json!({"db": 2, "host": "redis", "mode": "host", "port": 6380, "socket_timeout": 10.5, "socket_connect_timeout": 10.5}))), +)] +#[case::padded_environment_numbers_parse_like_python_int( + json!({}), + &[("REDIS_HOST", "redis"), ("REDIS_PORT", "6_380"), ("REDIS_DB", "3\n")], + Ok(expected_view(json!({"db": 3, "host": "redis", "mode": "host", "port": 6380}))), +)] +#[case::socket_timeout_from_the_environment( + json!({"host": "redis"}), + &[("REDIS_SOCKET_TIMEOUT", "0.75")], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379, "socket_connect_timeout": 0.75, "socket_timeout": 0.75}))), +)] +#[case::environment_url( + json!({}), + &[("REDIS_URL", "redis://environment:6379/1")], + Ok(expected_view(json!({"mode": "url", "url": "redis://environment:6379/1"}))), +)] +#[case::environment_url_beats_the_environment_host( + json!({}), + &[("REDIS_HOST", "ignored"), ("REDIS_URL", "redis://from-url:6379")], + Ok(expected_view(json!({"mode": "url", "url": "redis://from-url:6379"}))), +)] +#[case::an_explicit_host_drops_the_environment_url( + json!({"host": "configured"}), + &[("REDIS_URL", "redis://environment:6379")], + Ok(expected_view(json!({"db": 0, "host": "configured", "mode": "host", "port": 6379}))), +)] +#[case::a_url_drops_host_db_and_password_and_keeps_the_username( + json!({"db": 4, "host": "dropped", "password": "dropped", "url": "rediss://cache:6380", "username": "user"}), + &[], + Ok(expected_view(json!({"auth": {"kind": "static", "password": null, "username": "user"}, "mode": "url", "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}, "url": "rediss://cache:6380"}))), +)] +#[case::a_url_ignores_tls_settings_outside_the_url( + json!({"ssl_ca_certs": "/ca.pem", "ssl_cert_reqs": "none", "ssl_check_hostname": true, "url": "rediss://cache:6380"}), + &[], + Ok(expected_view(json!({"mode": "url", "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}, "url": "rediss://cache:6380"}))), +)] +#[case::startup_nodes_beat_a_url( + json!({"password": "cluster-secret", "startup_nodes": [{"host": "node-a", "port": 7000}], "url": "redis://single:6379"}), + &[], + Ok(expected_view(json!({"auth": {"kind": "static", "password": "cluster-secret", "username": null}, "mode": "cluster", "nodes": ["node-a:7000"]}))), +)] +#[case::environment_cluster_nodes( + json!({}), + &[("REDIS_CLUSTER_NODES", r#"[{"host": "node-a", "port": "7000"}, {"host": "node-b", "port": 7001}]"#)], + Ok(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000", "node-b:7001"]}))), +)] +#[case::cluster_tls_options( + json!({"ssl": "true", "ssl_ca_certs": "/ca.pem", "ssl_check_hostname": true, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Ok(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"], "tls": {"ca_certs": "/ca.pem", "certfile": null, "check_hostname": true, "keyfile": null, "verify_certificate": true}}))), +)] +#[case::sentinel_from_settings( + json!({"password": "data-secret", "sentinel_nodes": [["sentinel-a", 26379], ["sentinel-b", 26380]], "sentinel_password": "sentinel-secret", "service_name": "primary"}), + &[], + Ok(expected_view(json!({"auth": {"kind": "static", "password": "data-secret", "username": null}, "db": 0, "mode": "sentinel", "nodes": ["sentinel-a:26379", "sentinel-b:26380"], "sentinel_password": "sentinel-secret", "service_name": "primary"}))), +)] +#[case::sentinel_from_the_environment( + json!({}), + &[("REDIS_SENTINEL_NODES", r#"[["sentinel-a", 26379]]"#), ("REDIS_SENTINEL_PASSWORD", "sentinel-secret"), ("REDIS_SERVICE_NAME", "primary")], + Ok(expected_view(json!({"db": 0, "mode": "sentinel", "nodes": ["sentinel-a:26379"], "sentinel_password": "sentinel-secret", "service_name": "primary"}))), +)] +#[case::a_host_breaks_sentinel( + json!({"host": "ignored", "sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary"}), + &[], + Err("host cannot be combined with sentinel_nodes, which redis-py rejects at startup"), +)] +#[case::a_url_beats_sentinels( + json!({"sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary", "url": "redis://single:6379"}), + &[], + Ok(expected_view(json!({"mode": "url", "url": "redis://single:6379"}))), +)] +#[case::an_explicit_credential_provider_beats_cloud_auth( + json!({"aws_iam_auth": true, "credential_provider": true, "gcp_service_account": "account", "host": "redis", "password": "dropped"}), + &[("REDIS_AZURE_AD_TOKEN", "true")], + Ok(expected_view(json!({"auth": {"kind": "host"}, "db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::gcp_beats_azure_and_aws( + json!({"aws_iam_auth": true, "azure_redis_ad_token": "True", "host": "redis", "password": "dropped"}), + &[("REDIS_GCP_SERVICE_ACCOUNT", "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com")], + Ok(expected_view(json!({"auth": {"kind": "gcp_iam", "service_account": "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com"}, "db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::gcp_ca_certs_replace_the_configured_ones_under_ssl( + json!({"gcp_service_account": "account", "gcp_ssl_ca_certs": "/gcp-ca.pem", "host": "redis", "ssl": true, "ssl_ca_certs": "/configured-ca.pem"}), + &[], + Ok(expected_view(json!({"auth": {"kind": "gcp_iam", "service_account": "account"}, "db": 0, "host": "redis", "mode": "host", "port": 6379, "tls": {"ca_certs": "/gcp-ca.pem", "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}}))), +)] +#[case::azure_beats_aws_and_takes_the_environment_username( + json!({"aws_iam_auth": true, "azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis", "username": "dropped"}), + &[("REDIS_USERNAME", "environment-user")], + Ok(expected_view(json!({"auth": {"kind": "azure_ad", "username": "environment-user"}, "db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::azure_without_a_username_sends_only_the_token( + json!({"azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis"}), + &[], + Ok(expected_view(json!({"auth": {"kind": "azure_ad", "username": null}, "db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::azure_needs_the_literal_true( + json!({"azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": "1", "azure_tenant_id": "tenant", "host": "redis"}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::aws_iam_over_tls_with_the_default_region( + json!({"aws_iam_auth": "yes", "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "password": "dropped", "ssl": true}), + &[("AWS_DEFAULT_REGION", "us-east-1")], + Ok(expected_view(json!({"auth": {"cache_name": "cache-1", "kind": "aws_iam", "region": "us-east-1", "serverless": false, "user_name": "iam-user"}, "db": 0, "host": "cache-1", "mode": "host", "port": 6379, "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}}))), +)] +#[case::aws_region_beats_the_default_region( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "ssl": true}), + &[("AWS_DEFAULT_REGION", "us-east-1"), ("AWS_REGION", "eu-central-1")], + Ok(expected_view(json!({"auth": {"cache_name": "cache-1", "kind": "aws_iam", "region": "eu-central-1", "serverless": false, "user_name": "iam-user"}, "db": 0, "host": "cache-1", "mode": "host", "port": 6379, "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}}))), +)] +#[case::aws_iam_serverless_over_a_rediss_url_drops_its_userinfo( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_region": "eu-west-1", "aws_iam_serverless": true, "aws_iam_user_name": "iam-user", "url": "rediss://user:pass@cache-1:6379"}), + &[], + Ok(expected_view(json!({"auth": {"cache_name": "cache-1", "kind": "aws_iam", "region": "eu-west-1", "serverless": true, "user_name": "iam-user"}, "mode": "url", "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}, "url": "rediss://cache-1:6379"}))), +)] +#[case::aws_iam_without_tls_fails( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_region": "us-east-1", "aws_iam_user_name": "iam-user", "host": "cache-1"}), + &[], + Err("AWS ElastiCache IAM Redis authentication requires TLS"), +)] +#[case::aws_iam_without_a_region_fails( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "ssl": true}), + &[], + Err("AWS ElastiCache IAM Redis authentication requires: aws_iam_region"), +)] +#[case::tls_options_from_settings( + json!({"host": "redis", "ssl": true, "ssl_ca_certs": "/ca.pem", "ssl_cert_reqs": "none", "ssl_check_hostname": true}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379, "tls": {"ca_certs": "/ca.pem", "certfile": null, "check_hostname": true, "keyfile": null, "verify_certificate": false}}))), +)] +#[case::tls_options_from_the_environment( + json!({"host": "redis"}), + &[("REDIS_SSL", "true"), ("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional")], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379, "tls": {"ca_certs": "/env-ca.pem", "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}}))), +)] +#[case::connect_timeout_is_separate_from_the_socket_timeout( + json!({"host": "redis", "socket_connect_timeout": 0.25, "socket_timeout": 2.5}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379, "socket_connect_timeout": 0.25, "socket_timeout": 2.5}))), +)] +#[case::environment_references_resolve( + json!({"host": "os.environ/COORD_HOST", "password": "os.environ/COORD_PASSWORD"}), + &[("COORD_HOST", "referenced"), ("COORD_PASSWORD", "referenced-secret")], + Ok(expected_view(json!({"auth": {"kind": "static", "password": "referenced-secret", "username": null}, "db": 0, "host": "referenced", "mode": "host", "port": 6379}))), +)] +#[case::no_target_fails( + json!({"port": 6379}), + &[], + Err("either host or url must be specified for redis"), +)] +#[case::client_certificates( + json!({"host": "redis", "ssl": true, "ssl_certfile": "/client.pem", "ssl_keyfile": "/client.key"}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379, "tls": {"ca_certs": null, "certfile": "/client.pem", "check_hostname": false, "keyfile": "/client.key", "verify_certificate": true}}))), +)] +#[case::a_rediss_url_takes_tls_options_from_its_query( + json!({"url": "rediss://cache:6380/0?ssl_cert_reqs=none&ssl_ca_certs=/ca.pem&ssl_check_hostname=yes"}), + &[], + Ok(expected_view(json!({"mode": "url", "tls": {"ca_certs": "/ca.pem", "certfile": null, "check_hostname": true, "keyfile": null, "verify_certificate": false}, "url": "rediss://cache:6380/0?ssl_cert_reqs=none&ssl_ca_certs=/ca.pem&ssl_check_hostname=yes"}))), +)] +#[case::a_url_query_setting_without_a_redis_rs_equivalent( + json!({"url": "redis://cache:6379/0?socket_timeout=2"}), + &[], + Err("Redis host does not yet support configured setting url ?socket_timeout"), +)] +#[case::tls_settings_outside_a_url_never_fail_it( + json!({"db": "99999999999999999999", "ssl_cert_reqs": "strict", "url": "rediss://cache:6380"}), + &[], + Ok(expected_view(json!({"mode": "url", "tls": {"ca_certs": null, "certfile": null, "check_hostname": false, "keyfile": null, "verify_certificate": true}, "url": "rediss://cache:6380"}))), +)] +#[case::db_is_not_read_for_a_cluster( + json!({"db": 99_999_999_999_i64, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Err("db must be 0 in cluster mode"), +)] +#[case::cluster_db_true( + json!({"db": true, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Err("db must be 0 in cluster mode"), +)] +#[case::cluster_db_false( + json!({"db": false, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Ok(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"]}))), +)] +#[case::cluster_db_fraction( + json!({"db": 0.5, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Err("db must be 0 in cluster mode"), +)] +#[case::cluster_db_float_zero( + json!({"db": 0.0, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Ok(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"]}))), +)] +#[case::cluster_db_nonempty_list( + json!({"db": [1], "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Err("db must be 0 in cluster mode"), +)] +#[case::cluster_db_empty_list( + json!({"db": [], "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + Ok(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"]}))), +)] +#[case::ssl_ciphers( + json!({"host": "redis", "ssl": true, "ssl_ciphers": "HIGH"}), + &[], + Err("Redis host does not yet support configured setting ssl_ciphers"), +)] +#[case::retry_on_timeout( + json!({"host": "redis", "retry_on_timeout": true}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::malformed_port( + json!({"host": "redis", "port": "abc"}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::malformed_db( + json!({"db": "first", "host": "redis"}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::socket_timeout_not_a_number( + json!({"host": "redis", "socket_timeout": "slow"}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::malformed_port_from_the_environment( + json!({}), + &[("REDIS_HOST", "redis"), ("REDIS_PORT", "abc")], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::protocol_3( + json!({"host": "redis", "protocol": 3}), + &[], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::retry_on_error_from_the_environment( + json!({"host": "redis"}), + &[("REDIS_RETRY_ON_ERROR", "true")], + Ok(expected_view(json!({"db": 0, "host": "redis", "mode": "host", "port": 6379}))), +)] +#[case::unix_socket_path( + json!({"host": "redis", "unix_socket_path": "/tmp/redis.sock"}), + &[], + Err("Redis host does not yet support configured setting unix_socket_path"), +)] +fn resolves_like_python_redis( + #[case] kwargs: serde_json::Value, + #[case] environment: &[(&str, &str)], + #[case] expected: Result, +) { + let environment: HashMap = environment + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + + let outcome = resolve_kwargs(kwargs, &environment) + .map(|resolved| view(&resolved)) + .map_err(|error| error.to_string()); + + assert_eq!(outcome, expected.map_err(str::to_owned)); +} + +fn snapshot(values: &[(&str, &str)]) -> SecretSnapshot { + let values: HashMap<&str, &str> = values.iter().copied().collect(); + SecretSnapshot::read( + ConfiguredRedis::environment_only().secret_names(), + &|name: &str| values.get(name).map(|value| (*value).to_owned()), + ) +} + +#[rstest] +fn secret_manager_values_reach_the_resolver_through_the_snapshot() { + let settings = redis_cache_kwargs(json!({"host": "os.environ/COORD_HOST"})); + let secrets = SecretSnapshot::read(settings.secret_names(), &|name: &str| match name { + "COORD_HOST" => Some("from-the-manager".to_owned()), + "REDIS_PASSWORD" => Some("manager-secret".to_owned()), + _ => None, + }); + + let resolved = settings.resolve(&secrets, &|_: &str| None).unwrap().redis; + + assert!(matches!(&resolved.endpoint, Endpoint::Host(host) if host.host == "from-the-manager")); + assert_eq!(resolved.auth, static_auth(None, Some("manager-secret"))); +} + +#[rstest] +#[case::from_the_environment(&[], &[("REDIS_USERNAME", "environment-user")], Some("environment-user"))] +#[case::never_from_the_secret_manager(&[("REDIS_USERNAME", "manager-user")], &[], None)] +fn the_azure_username_comes_from_the_process_environment_like_python( + #[case] secrets: &[(&str, &str)], + #[case] environment: &[(&str, &str)], + #[case] expected: Option<&str>, +) { + let settings = redis_cache_kwargs(json!({"host": "redis", "azure_redis_ad_token": "true"})); + let environment: HashMap<&str, &str> = environment.iter().copied().collect(); + + let resolved = settings + .resolve(&snapshot(secrets), &|name: &str| { + environment.get(name).map(|value| (*value).to_owned()) + }) + .unwrap() + .redis; + + assert!( + matches!(&resolved.auth, RedisAuth::AzureAd(azure) if azure.username.as_deref() == expected), + "{:?}", + resolved.auth + ); +} + +#[rstest] +#[case::coordination_drops_the_null( + "general_settings:\n coordination_redis:\n host: redis\n ssl: null", + false, + true +)] +#[case::cache_passes_an_unnamed_null_on( + "litellm_settings:\n cache_params:\n type: redis\n host: redis\n ssl: null", + true, + false +)] +#[case::cache_drops_a_null_it_names( + "litellm_settings:\n cache_params:\n type: redis\n host: null\n ssl: true", + true, + true +)] +fn a_null_overrides_the_environment_only_where_python_passes_it_on( + #[case] yaml: &str, + #[case] cache: bool, + #[case] tls: bool, +) { + let environment = [("REDIS_HOST", "environment"), ("REDIS_SSL", "true")]; + let resolved = match cache { + true => resolve_cache_in(yaml, &environment), + false => resolve(yaml, &environment), + } + .unwrap(); + + let Endpoint::Host(host) = resolved.endpoint else { + panic!("expected a host endpoint, got {:?}", resolved.endpoint); + }; + assert_eq!(host.tls.is_some(), tls); } diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 6eeaa43e013..7434ac603fc 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -19,9 +19,13 @@ futures-util.workspace = true litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true litellm-config.workspace = true +litellm-core-utils.workspace = true litellm-error.workspace = true litellm-circuit-breaker.workspace = true litellm-redis-client.workspace = true +litellm-redis-connect.workspace = true +litellm-redis-types.workspace = true +rustls.workspace = true litellm-http.workspace = true litellm-llms.workspace = true litellm-secrets.workspace = true diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index e2f646c5448..d03d7c353bf 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -12,13 +12,10 @@ pub enum Error { Mcp(#[from] litellm_gateway_mcp::ConnectError), #[error(transparent)] Config(#[from] litellm_config::Error), - #[error("invalid Redis setting {setting}: {reason}")] - RedisInvalid { - setting: &'static str, - reason: &'static str, - }, #[error(transparent)] - RedisConnect(#[from] litellm_redis_client::ConnectError), + Redis(#[from] litellm_config::RedisError), + #[error(transparent)] + RedisConnect(#[from] litellm_redis_connect::Error), #[error("MCP requires working gateway authentication")] Auth(#[from] litellm_gateway_auth::Error), } diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 12a9bf9110a..ed6cd64d69c 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -68,6 +68,12 @@ pub fn build_inference(config: &Config) -> Result, Error> { Ok(Arc::new(Gateway::new(resources, http, secrets, router)?)) } +pub fn install_crypto_provider() { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } +} + pub async fn build_redis(config: &Config) -> Result { let environment = secrets::environment_values(&config.environment_variables)?; let lookup = |name: &str| { diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 47c1fe83f50..589103191ae 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -41,6 +41,7 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { + litellm_gateway::install_crypto_provider(); let settings: Settings = settings::from_iter(std::env::vars_os())?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; diff --git a/litellm-rust/crates/gateway/src/redis.rs b/litellm-rust/crates/gateway/src/redis.rs index d0e9a838232..cfe67c04f03 100644 --- a/litellm-rust/crates/gateway/src/redis.rs +++ b/litellm-rust/crates/gateway/src/redis.rs @@ -2,13 +2,12 @@ use std::time::Duration; use litellm_circuit_breaker::{Breaker, Config as BreakerConfig, State}; use litellm_config::{ - CacheSettings, Config, Lookup, RedisEndpoint, RedisSettings, ResolvedRedis, Value, - environment_has_target, -}; -use litellm_redis_client::{ - ClusterEndpoint, ClusterNode, DEFAULT_TIMEOUT, Endpoint, HostEndpoint, Namespace, RedisClient, - RedisConfig, Timeouts, + CacheSettings, Config, ConfiguredRedis, Expected, RedisError, RedisSettings, Resolution, + SecretSnapshot, Value, environment_has_target, }; +use litellm_core_utils::settings::Lookup; +use litellm_redis_client::RedisClient; +use litellm_redis_types::ResolvedRedis; use crate::Error; @@ -34,12 +33,15 @@ enum Target { Environment(Params), } -pub(crate) fn plan(config: &Config, lookup: Lookup<'_>) -> Result { +pub(crate) fn plan(config: &Config, lookup: &dyn Lookup) -> Result { let response_cache = redis_cache_params(config) - .map(|settings| settings.resolve_cache(lookup)) + .map(|settings| resolve(&settings.cache_params(), lookup)) .transpose()?; let coordination = match &config.general_settings.coordination_redis { - Some(settings) => Some(Target::Configured(settings.resolve(lookup)?)), + Some(settings) => Some(Target::Configured(resolve( + &settings.coordination()?, + lookup, + )?)), None => match (&response_cache, config.litellm_settings.cache == Some(true)) { (Some(cache), _) => Some(Target::Configured(cache.clone())), (None, true) => environment(lookup)?.map(Target::Configured), @@ -57,9 +59,15 @@ pub(crate) fn plan(config: &Config, lookup: Lookup<'_>) -> Result { }) } -pub async fn connect(config: &Config, lookup: Lookup<'_>) -> Result { +pub async fn connect(config: &Config, lookup: &dyn Lookup) -> Result { let plan = plan(config, lookup)?; let breaker = breaker_config(lookup)?; + let client = |params: Params, breaker: Option| -> Result { + Ok(litellm_redis_connect::connect( + params, + breaker.map(|config| Breaker::new(config, Box::new(log_transition))), + )?) + }; let coordination = match plan.coordination { Some(Target::Configured(params)) => Some(client(params, breaker)?), Some(Target::Environment(params)) => match client(params, breaker) { @@ -98,17 +106,23 @@ fn redis_cache_params(config: &Config) -> Option<&RedisSettings> { } } -fn environment(lookup: Lookup<'_>) -> Result, Error> { +fn resolve(configured: &ConfiguredRedis, lookup: &dyn Lookup) -> Result { + let secrets = SecretSnapshot::read(configured.secret_names(), lookup); + let Resolution { redis, ignored } = configured.resolve(&secrets, lookup)?; + for setting in ignored { + tracing::warn!(%setting, "Redis setting is accepted but ignored"); + } + Ok(redis) +} + +fn environment(lookup: &dyn Lookup) -> Result, Error> { match environment_has_target(lookup) { - true => RedisSettings::default() - .resolve(lookup) - .map(Some) - .map_err(Into::into), + true => Ok(Some(resolve(&ConfiguredRedis::environment_only(), lookup)?)), false => Ok(None), } } -fn optional_environment(lookup: Lookup<'_>) -> Option { +fn optional_environment(lookup: &dyn Lookup) -> Option { match environment(lookup) { Ok(params) => params, Err(error) => { @@ -149,41 +163,6 @@ async fn reachable(client: RedisClient) -> Option { } } -fn client(params: Params, breaker: Option) -> Result { - tracing::info!(endpoint = ?params.endpoint, namespace = ?params.namespace, "redis client configured"); - let endpoint = match params.endpoint { - RedisEndpoint::Url(url) => Endpoint::Url(url), - RedisEndpoint::Host(host) => Endpoint::Host(HostEndpoint { - host: host.host, - port: host.port, - username: host.username, - password: host.password, - tls: host.tls, - db: host.db, - }), - RedisEndpoint::Cluster(cluster) => Endpoint::Cluster(ClusterEndpoint { - nodes: cluster - .nodes - .into_iter() - .map(|node| ClusterNode { - host: node.host, - port: node.port, - }) - .collect(), - username: cluster.username, - password: cluster.password, - tls: cluster.tls, - }), - }; - Ok(RedisClient::new(RedisConfig { - endpoint, - namespace: params.namespace.map(Namespace::new).unwrap_or_default(), - timeouts: Timeouts::new(params.timeout.unwrap_or(DEFAULT_TIMEOUT), None), - breaker: breaker.map(|config| Breaker::new(config, Box::new(log_transition))), - events: None, - })?) -} - fn log_transition(transition: &litellm_circuit_breaker::Transition) { match transition.to { State::Open { .. } => tracing::warn!( @@ -201,54 +180,70 @@ fn log_transition(transition: &litellm_circuit_breaker::Transition) { } } -fn breaker_config(lookup: Lookup<'_>) -> Result, Error> { - let enabled = lookup("REDIS_CIRCUIT_BREAKER_ENABLED") +fn parse_seconds(text: &str) -> Option { + text.parse::() + .ok() + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) +} + +fn breaker_config(lookup: &dyn Lookup) -> Result, Error> { + let enabled = lookup + .get("REDIS_CIRCUIT_BREAKER_ENABLED") .is_none_or(|value| value.eq_ignore_ascii_case("true")); if !enabled { return Ok(None); } let defaults = BreakerConfig::default(); - let seconds = |name: &'static str, default: Duration| -> Result { - lookup(name) - .map(|value| { - value - .parse::() - .ok() - .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) - .ok_or(Error::RedisInvalid { - setting: name, - reason: "must be a non-negative number of seconds", - }) - }) - .unwrap_or(Ok(default)) - }; - let failure_threshold = lookup("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD") + Ok(Some(BreakerConfig { + failure_threshold: environment_value( + lookup, + "REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", + Expected::Integer, + |text| text.parse().ok(), + )? + .unwrap_or(defaults.failure_threshold), + recovery_timeout: environment_value( + lookup, + "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", + Expected::Seconds, + parse_seconds, + )? + .unwrap_or(defaults.recovery_timeout), + timeout_min_duration: environment_value( + lookup, + "REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", + Expected::Seconds, + parse_seconds, + )? + .unwrap_or(defaults.timeout_min_duration), + })) +} + +fn environment_value( + lookup: &dyn Lookup, + name: &str, + expected: Expected, + parse: impl Fn(&str) -> Option, +) -> Result, RedisError> { + lookup + .get(name) .map(|value| { - value.parse().map_err(|_| Error::RedisInvalid { - setting: "REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", - reason: "must be an integer", + parse(&value).ok_or_else(|| RedisError::Invalid { + setting: name.to_owned(), + expected, }) }) - .unwrap_or(Ok(defaults.failure_threshold))?; - Ok(Some(BreakerConfig { - failure_threshold, - recovery_timeout: seconds( - "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", - defaults.recovery_timeout, - )?, - timeout_min_duration: seconds( - "REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", - defaults.timeout_min_duration, - )?, - })) + .transpose() } #[cfg(test)] mod tests { use std::{collections::HashMap, time::Duration}; - use litellm_config::{Config, RedisEndpoint, RedisHost}; + use litellm_config::{Config, RedisError}; + use litellm_redis_client::{Endpoint, HostEndpoint}; use litellm_redis_testing::RedisServer; + use litellm_redis_types::RedisAuth; use rstest::rstest; use super::{Params, Plan, Target, connect, plan}; @@ -256,16 +251,19 @@ mod tests { fn host(name: &str, port: u16) -> Params { Params { - endpoint: RedisEndpoint::Host(RedisHost { + endpoint: Endpoint::Host(HostEndpoint { host: name.to_owned(), port, - username: None, - password: None, - tls: false, + tls: None, db: 0, }), + auth: RedisAuth::Static { + username: None, + password: None, + }, namespace: None, timeout: None, + connect_timeout: None, } } @@ -274,7 +272,7 @@ mod tests { .iter() .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) .collect(); - plan(&Config::from_yaml(yaml).unwrap(), &|name| { + plan(&Config::from_yaml(yaml).unwrap(), &|name: &str| { environment.get(name).cloned() }) } @@ -351,11 +349,11 @@ mod tests { port_of(&server) )) .unwrap(); - let clients = connect(&config, &|_| None).await.unwrap(); + let clients = connect(&config, &|_: &str| None).await.unwrap(); let coordination = clients.coordination.unwrap(); let buffer = clients.transaction_buffer.unwrap(); let cache = clients.response_cache.unwrap(); - let unnamespaced = connect(&coordination_config(&server), &|_| None) + let unnamespaced = connect(&coordination_config(&server), &|_: &str| None) .await .unwrap() .coordination @@ -405,7 +403,6 @@ mod tests { } #[rstest] - #[case::malformed_port(&[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")])] #[case::malformed_cluster_nodes(&[("REDIS_CLUSTER_NODES", "[{\"host\": ")])] fn redis_migration_regression_malformed_optional_environment_is_absent( #[case] environment: &[(&str, &str)], @@ -414,54 +411,45 @@ mod tests { } #[rstest] - fn redis_migration_regression_explicit_malformed_setting_still_fails() { + #[case::optional_environment("model_list: []", &[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")])] + #[case::transaction_buffer_environment( + "general_settings:\n use_redis_transaction_buffer: true", + &[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")] + )] + #[case::response_cache( + "litellm_settings:\n cache: true\n cache_params:\n type: redis\n host: env\n port: not-a-port", + &[] + )] + fn a_malformed_port_outside_the_coordination_block_falls_back_to_6379_like_redis_py( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + ) { + let plan = resolved(yaml, environment).unwrap(); + + let target = match (plan.coordination, plan.transaction_buffer_fallback) { + (Some(Target::Configured(params) | Target::Environment(params)), _) + | (None, Some(params)) => params, + (None, None) => panic!("expected a Redis target"), + }; + assert_eq!(target.endpoint, host("env", 6379).endpoint); + } + + #[rstest] + #[case::port("port: not-a-port", "port")] + #[case::ssl("ssl: maybe", "ssl")] + fn a_malformed_coordination_block_fails_like_the_pydantic_model( + #[case] setting: &str, + #[case] expected: &str, + ) { let error = resolved( - "general_settings:\n coordination_redis:\n host: coord\n port: not-a-port", + &format!("general_settings:\n coordination_redis:\n host: coord\n {setting}"), &[], ) .unwrap_err(); - assert!(matches!( - error, - Error::Config(litellm_config::Error::RedisInvalid { - setting: "port", - .. - }) - )); - } - - #[rstest] - fn redis_migration_regression_enabled_transaction_buffer_rejects_invalid_environment() { - let error = resolved( - "general_settings:\n use_redis_transaction_buffer: true", - &[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")], - ) - .unwrap_err(); - - assert!(matches!( - error, - Error::Config(litellm_config::Error::RedisInvalid { - setting: "port", - .. - }) - )); - } - - #[rstest] - fn redis_migration_regression_invalid_response_cache_still_fails() { - let error = resolved( - "litellm_settings:\n cache: true\n cache_params:\n type: redis\n host: cache\n port: not-a-port", - &[], - ) - .unwrap_err(); - - assert!(matches!( - error, - Error::Config(litellm_config::Error::RedisInvalid { - setting: "port", - .. - }) - )); + assert!( + matches!(error, Error::Redis(RedisError::Invalid { setting, .. }) if setting == expected) + ); } #[rstest] @@ -469,7 +457,7 @@ mod tests { async fn redis_migration_regression_invalid_optional_environment_url_is_absent() { let config = Config::from_yaml("model_list: []").unwrap(); - let clients = connect(&config, &|name| { + let clients = connect(&config, &|name: &str| { (name == "REDIS_URL").then(|| "not a redis URL".to_owned()) }) .await @@ -507,7 +495,7 @@ mod tests { let server = RedisServer::start().await.unwrap(); server.stop().await.unwrap(); - let clients = connect(&coordination_config(&server), &|_| None) + let clients = connect(&coordination_config(&server), &|_: &str| None) .await .unwrap(); let coordination = clients.coordination.unwrap(); @@ -522,7 +510,7 @@ mod tests { #[tokio::test] async fn the_coordination_client_survives_a_redis_restart_and_backs_the_transaction_buffer() { let server = RedisServer::start().await.unwrap(); - let clients = connect(&coordination_config(&server), &|_| None) + let clients = connect(&coordination_config(&server), &|_: &str| None) .await .unwrap(); let coordination = clients.coordination.unwrap(); @@ -538,6 +526,30 @@ mod tests { assert!(buffer.ping().await.is_ok()); } + #[rstest] + #[tokio::test] + async fn an_environment_only_tls_redis_is_reached_with_its_ca() { + crate::install_crypto_provider(); + let server = litellm_redis_testing::RedisTlsServer::start(false) + .await + .unwrap(); + let ca = server.files().ca.display().to_string(); + let port = server.port().to_string(); + let config = Config::from_yaml("model_list: []").unwrap(); + + let clients = connect(&config, &|name: &str| match name { + "REDIS_HOST" => Some(server.host().to_owned()), + "REDIS_PORT" => Some(port.clone()), + "REDIS_SSL" => Some("true".to_owned()), + "REDIS_SSL_CA_CERTS" => Some(ca.clone()), + _ => None, + }) + .await + .unwrap(); + + assert!(clients.coordination.is_some()); + } + #[rstest] #[tokio::test] async fn an_unreachable_environment_only_redis_leaves_coordination_in_memory() { @@ -546,7 +558,7 @@ mod tests { server.stop().await.unwrap(); let config = Config::from_yaml("model_list: []").unwrap(); - let clients = connect(&config, &|name| match name { + let clients = connect(&config, &|name: &str| match name { "REDIS_HOST" => Some("127.0.0.1".to_owned()), "REDIS_PORT" => Some(port.clone()), _ => None, @@ -565,7 +577,7 @@ mod tests { let port = port_of(&server); let config = Config::from_yaml("model_list: []").unwrap(); - let clients = connect(&config, &|name| match name { + let clients = connect(&config, &|name: &str| match name { "REDIS_HOST" => Some("127.0.0.1".to_owned()), "REDIS_PORT" => Some(port.clone()), _ => None, diff --git a/litellm-rust/crates/python-compat/src/lib.rs b/litellm-rust/crates/python-compat/src/lib.rs index b23aff409f2..e15141195f5 100644 --- a/litellm-rust/crates/python-compat/src/lib.rs +++ b/litellm-rust/crates/python-compat/src/lib.rs @@ -10,6 +10,7 @@ //! | [`json`] | `json.dumps(value)`, and `json.loads(json.dumps(value))` as a JSON value | //! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | //! | [`truthy`] | `bool(value)` | +//! | [`number`] | `int(text)` and `float(text)` | //! //! [`Value`] is the closed data model these formats share. Live Python objects //! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the @@ -20,10 +21,12 @@ //! (string hashes are randomized per process). Sets keep their literal order. //! - `str` values are Rust `String`s, so lone surrogates cannot be represented. //! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists. +//! - [`number`] reads ASCII digits only, while Python's `int` and `float` accept any Unicode decimal digit. mod error; pub mod json; pub mod literal; +pub mod number; pub mod pickle; pub mod repr; pub mod truthy; diff --git a/litellm-rust/crates/python-compat/src/number.rs b/litellm-rust/crates/python-compat/src/number.rs new file mode 100644 index 00000000000..3286e7267be --- /dev/null +++ b/litellm-rust/crates/python-compat/src/number.rs @@ -0,0 +1,82 @@ +use std::str::FromStr; + +use num_bigint::BigInt; + +/// `int(text)` in base 10, or `None` where Python raises `ValueError`. +pub fn int(text: &str) -> Option { + parse(text) +} + +/// `float(text)`, or `None` where Python raises `ValueError`. +pub fn float(text: &str) -> Option { + parse(text) +} + +fn parse(text: &str) -> Option { + let text = text.trim(); + let digit_at = |index: Option| { + index + .and_then(|index| text.as_bytes().get(index)) + .is_some_and(u8::is_ascii_digit) + }; + let underscores_between_digits = text + .match_indices('_') + .all(|(index, _)| digit_at(index.checked_sub(1)) && digit_at(Some(index + 1))); + if !underscores_between_digits { + return None; + } + text.replace('_', "").parse().ok() +} + +#[cfg(test)] +mod tests { + use num_bigint::BigInt; + use rstest::rstest; + + use super::{float, int}; + + // Expected values are CPython 3.14 int(text) + #[rstest] + #[case::trailing_newline("2\n", Some(2))] + #[case::surrounding_spaces_and_underscore(" 6_380 ", Some(6380))] + #[case::plus_sign("+5", Some(5))] + #[case::negative_zero("-0", Some(0))] + #[case::leading_zeros("007", Some(7))] + #[case::unicode_spaces("\u{a0}7\u{2003}", Some(7))] + #[case::beyond_i64("99999999999999999999", Some(99_999_999_999_999_999_999_i128))] + #[case::double_underscore("1__0", None)] + #[case::leading_underscore("_1", None)] + #[case::trailing_underscore("1_", None)] + #[case::decimal_point("5.0", None)] + #[case::information_separator("\x1c7", None)] + #[case::empty("", None)] + #[case::sign_only("+", None)] + #[case::non_ascii_digit_is_a_known_limit("\u{667}", None)] + fn int_parses_like_python(#[case] text: &str, #[case] expected: Option) { + assert_eq!(int(text), expected.map(BigInt::from)); + } + + // Expected values are CPython 3.14 float(text) + #[rstest] + #[case::underscore_and_newline(" 1_0.5\n", Some(10.5))] + #[case::exponent("1e3", Some(1000.0))] + #[case::leading_point(".5", Some(0.5))] + #[case::trailing_point("5.", Some(5.0))] + #[case::plus_sign("+1.5", Some(1.5))] + #[case::infinity_word("Infinity", Some(f64::INFINITY))] + #[case::infinity_with_space("infinity ", Some(f64::INFINITY))] + #[case::underscore_before_exponent("1_e3", None)] + #[case::underscore_after_exponent("1e_3", None)] + #[case::underscore_after_point("1._5", None)] + #[case::underscore_after_word("inf_", None)] + #[case::hexadecimal("0x10", None)] + #[case::bare_exponent("1.5e", None)] + fn float_parses_like_python(#[case] text: &str, #[case] expected: Option) { + assert_eq!(float(text), expected); + } + + #[rstest] + fn float_reads_a_signed_nan_like_python() { + assert!(float("-NaN").is_some_and(f64::is_nan)); + } +} diff --git a/litellm-rust/crates/redis-client/AGENTS.md b/litellm-rust/crates/redis-client/AGENTS.md index 9620735e7f6..abb39fb4cd4 100644 --- a/litellm-rust/crates/redis-client/AGENTS.md +++ b/litellm-rust/crates/redis-client/AGENTS.md @@ -6,7 +6,7 @@ ## Errors - `Transport`, `Server`, `Decode`, each wrapping the redis-rs error as `#[source]`. `ConnectError` is separate because only `connect` returns it -- Classify with `Error::classify(error, deployment)`, never a bare `From`, because redis-py (5.3.1) decides by topology. `Transport` is what Python's `_redis_health_error_types` counts: LOADING everywhere, CLUSTERDOWN and MASTERDOWN only on cluster (only `ClusterParser` maps them), plus NOAUTH, WRONGPASS and the `ERR` messages redis-py maps to `ConnectionError`, matched exactly like `parse_error`. TRYAGAIN is `Server`, since the cluster client already retried it. Each code gets its own rstest case built from a parsed RESP error +- Classify with `Error::classify(error, deployment)`, never a bare `From`, because redis-py (5.3.1) decides by topology. `Transport` is what Python's `_redis_health_error_types` counts: LOADING everywhere, CLUSTERDOWN and MASTERDOWN only on cluster (only `ClusterParser` maps them), READONLY only under Sentinel (where `SentinelManagedConnection` turns it into `ConnectionError` and rediscovers), plus NOAUTH, WRONGPASS and the `ERR` messages redis-py maps to `ConnectionError`, matched exactly like `parse_error`. TRYAGAIN is `Server`, since the cluster client already retried it. Each code gets its own rstest case built from a parsed RESP error - A `Server` error never drops the connection - `Transport` includes timeouts, where the command may still have applied. Callers of counters and locks treat it as an unknown outcome - Never swallow an error or turn it into a miss. The caller picks the fallback @@ -45,3 +45,11 @@ - Every command test runs on each topology through `#[values(Topology::Standalone, Topology::Cluster)]`. A test is single-topology only when the behavior exists on one (a `db` number has no cluster meaning) - Assert values and Redis state. Outages come from the container (`restart`, `freeze` / `thaw`), not `DEBUG`, which the official image disables - Use a short client timeout in outage tests so they stay fast. Never sleep to wait for a state; poll with a deadline + +## TLS, credentials and Sentinel +- `TlsOptions` follows redis-py 5.3.1. It verifies the certificate by default, skips the hostname check by default (`ssl_check_hostname=False`), and trusts the system roots plus `ca_certs` +- redis-rs loads the system trust store for every TLS connection and fails if that fails, even with `ca_certs` set. Where the macOS keychain is blocked (agent sandboxes), run the TLS tests with `SSL_CERT_FILE=/etc/ssl/cert.pem` +- `Credentials` are static and sent at connect. URL userinfo wins over them, like redis-py's `from_url` +- Sentinel discovers the master on each new connection and drops it on a transport failure or READONLY, like redis-py's `SentinelConnectionPool` +- Sentinel monitors get `sentinel_password`, the data username when that password is set, and the TLS mode and certificates. Python's `_sentinel_auth_kwargs` forwards every other connection kwarg too, and it sends the username even without a sentinel password +- A `Debug` impl never shows a secret. Follow `.agents/skills/rust-redaction` diff --git a/litellm-rust/crates/redis-client/Cargo.toml b/litellm-rust/crates/redis-client/Cargo.toml index 4a5de772cfe..3a7699a70d8 100644 --- a/litellm-rust/crates/redis-client/Cargo.toml +++ b/litellm-rust/crates/redis-client/Cargo.toml @@ -8,12 +8,22 @@ repository.workspace = true [dependencies] litellm-circuit-breaker.workspace = true litellm-secret-value.workspace = true +litellm-redis-types.workspace = true futures-util.workspace = true -redis = { workspace = true, features = ["cluster-async", "connection-manager", "tokio-rustls-comp"] } +redis = { workspace = true, features = [ + "cluster-async", + "connection-manager", + "sentinel", + "tls-rustls-insecure", + "tokio-rustls-comp", +] } +pem = "4.0.0" +rustls-native-certs.workspace = true socket2 = "0.6.5" thiserror.workspace = true tokio = { workspace = true, features = ["sync", "time"] } [dev-dependencies] litellm-redis-testing.workspace = true +rustls.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/redis-client/src/client.rs b/litellm-rust/crates/redis-client/src/client.rs index a3382e6d5ef..d7fbdf5da21 100644 --- a/litellm-rust/crates/redis-client/src/client.rs +++ b/litellm-rust/crates/redis-client/src/client.rs @@ -5,13 +5,11 @@ use std::{ }; use litellm_circuit_breaker::{Admission, Breaker, Failure}; -use redis::{ - Cmd, FromRedisValue, IntoConnectionInfo, RedisError, ToRedisArgs, ToSingleRedisArg, Value, -}; +use redis::{Cmd, FromRedisValue, RedisError, ToRedisArgs, ToSingleRedisArg, Value}; use crate::{ - CallEvent, ConnectError, Endpoint, Error, Expiry, FailureKind, Namespace, Operation, Outcome, - Pipeline, RedisConfig, RedisEvents, Script, + CallEvent, ConnectError, Error, Expiry, FailureKind, Namespace, Operation, Outcome, Pipeline, + RedisConfig, RedisEvents, Script, commands::{self, ScriptCall}, connection::Connection, }; @@ -33,17 +31,8 @@ pub enum Ttl { impl RedisClient { pub fn new(config: RedisConfig) -> Result { - let connection = match &config.endpoint { - Endpoint::Url(url) => { - Connection::standalone(url.expose().into_connection_info()?, config.timeouts)? - } - Endpoint::Host(host) => { - Connection::standalone(host.connection_info()?, config.timeouts)? - } - Endpoint::Cluster(cluster) => Connection::cluster(cluster, config.timeouts)?, - }; Ok(Self { - connection, + connection: Connection::new(&config.endpoint, &config.credentials, config.timeouts)?, namespace: config.namespace, breaker: config.breaker, events: config.events, diff --git a/litellm-rust/crates/redis-client/src/config.rs b/litellm-rust/crates/redis-client/src/config.rs index dd94e509e89..fa4d1bcbe10 100644 --- a/litellm-rust/crates/redis-client/src/config.rs +++ b/litellm-rust/crates/redis-client/src/config.rs @@ -1,17 +1,22 @@ -use std::{sync::Arc, time::Duration}; +use std::{path::PathBuf, sync::Arc, time::Duration}; -use redis::{ConnectionAddr, ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisResult}; +use redis::{ + ClientTlsConfig, ConnectionAddr, ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, + TlsCertificates, TlsMode, io::tcp::TcpSettings, +}; use litellm_circuit_breaker::Breaker; use litellm_secret_value::SecretValue; -use crate::{Namespace, RedisEvents}; +use crate::{ConnectError, Namespace, RedisEvents}; +use litellm_redis_types::{ClusterEndpoint, Endpoint, TlsOptions}; // litellm/constants.py REDIS_SOCKET_TIMEOUT, which _coerce_redis_kwargs_types sets when socket_timeout is unset pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); pub struct RedisConfig { pub endpoint: Endpoint, + pub credentials: Credentials, pub namespace: Namespace, pub timeouts: Timeouts, pub breaker: Option, @@ -39,103 +44,144 @@ impl Default for Timeouts { } } -#[derive(Clone, Debug, PartialEq)] -pub enum Endpoint { - Url(SecretValue), - Host(HostEndpoint), - Cluster(ClusterEndpoint), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct HostEndpoint { - pub host: String, - pub port: u16, +#[derive(Clone, Debug)] +pub struct Credentials { pub username: Option, pub password: Option, - pub tls: bool, - pub db: i64, } -#[derive(Clone, Debug, PartialEq)] -pub struct ClusterEndpoint { - pub nodes: Vec, - pub username: Option, - pub password: Option, - pub tls: bool, +impl Credentials { + pub const NONE: Self = Self { + username: None, + password: None, + }; } -#[derive(Clone, Debug, PartialEq)] -pub struct ClusterNode { - pub host: String, - pub port: u16, -} - -impl ClusterEndpoint { - pub(crate) fn addrs(&self) -> impl Iterator + '_ { - self.nodes - .iter() - .map(|node| addr(&node.host, node.port, self.tls)) +pub(crate) fn tls_mode(tls: &TlsOptions) -> TlsMode { + match tls.verify_certificate { + true => TlsMode::Secure, + false => TlsMode::Insecure, } } -impl HostEndpoint { - pub(crate) fn connection_info(&self) -> RedisResult { - let settings = RedisConnectionInfo::default().set_db(self.db); - let settings = match &self.username { - Some(username) => settings.set_username(username.expose()), - None => settings, - }; - let settings = match &self.password { - Some(password) => settings.set_password(password.expose()), - None => settings, - }; - Ok(addr(&self.host, self.port, self.tls) - .into_connection_info()? - .set_redis_settings(settings)) - } +pub(crate) fn tls_certificates(tls: &TlsOptions) -> Result { + let client_tls = match (&tls.certfile, &tls.keyfile) { + (Some(certfile), keyfile) => Some(ClientTlsConfig { + client_cert: read(certfile)?, + client_key: read(keyfile.as_ref().unwrap_or(certfile))?, + }), + (None, _) => None, + }; + Ok(TlsCertificates { + client_tls, + root_cert: tls + .ca_certs + .as_ref() + .map(read) + .transpose()? + .map(with_system_roots), + }) } -fn addr(host: &str, port: u16, tls: bool) -> ConnectionAddr { +pub(crate) fn apply_tls( + tls: &TlsOptions, + info: ConnectionInfo, +) -> Result { + let tls_info = redis::Client::build_with_tls(info, tls_certificates(tls)?)? + .get_connection_info() + .clone(); + let mut addr = match tls_info.addr().clone() { + ConnectionAddr::TcpTls { + host, + port, + tls_params, + .. + } => ConnectionAddr::TcpTls { + host, + port, + insecure: !tls.verify_certificate, + tls_params, + }, + addr => addr, + }; + addr.set_danger_accept_invalid_hostnames(!tls.check_hostname); + Ok(tls_info.set_addr(addr)) +} + +fn with_system_roots(configured: Vec) -> Vec { + let system = rustls_native_certs::load_native_certs(); + let system_pem = system + .certs + .iter() + .map(|cert| pem::encode(&pem::Pem::new("CERTIFICATE", cert.to_vec()))); + configured + .into_iter() + .chain(b"\n".iter().copied()) + .chain(system_pem.flat_map(String::into_bytes)) + .collect() +} + +fn read(path: &PathBuf) -> Result, ConnectError> { + std::fs::read(path).map_err(|source| ConnectError::TlsFile { + path: path.display().to_string(), + source, + }) +} + +pub(crate) fn addr(host: &str, port: u16, tls: Option<&TlsOptions>) -> ConnectionAddr { match tls { - true => ConnectionAddr::TcpTls { + Some(tls) => ConnectionAddr::TcpTls { host: host.to_owned(), port, - insecure: false, + insecure: !tls.verify_certificate, tls_params: None, }, - false => ConnectionAddr::Tcp(host.to_owned(), port), + None => ConnectionAddr::Tcp(host.to_owned(), port), } } -#[cfg(test)] -mod tests { - use litellm_secret_value::SecretValue; - use rstest::rstest; +pub(crate) fn cluster_addrs( + endpoint: &ClusterEndpoint, +) -> impl Iterator + '_ { + endpoint + .nodes + .iter() + .map(|node| addr(&node.host, node.port, endpoint.tls.as_ref())) +} - use super::{ClusterEndpoint, Endpoint, HostEndpoint}; - - #[rstest] - #[case::url(Endpoint::Url(SecretValue::new("redis://credential-value@cache:6379/0")))] - #[case::host( - Endpoint::Host(HostEndpoint { - host: "cache".to_owned(), - port: 6379, - username: Some(SecretValue::new("credential-value")), - password: Some(SecretValue::new("credential-value")), - tls: false, - db: 0, - }) - )] - #[case::cluster(Endpoint::Cluster(ClusterEndpoint { - nodes: Vec::new(), - username: Some(SecretValue::new("credential-value")), - password: Some(SecretValue::new("credential-value")), - tls: false, - }))] - fn debug_output_never_shows_the_secret(#[case] endpoint: Endpoint) { - let debug = format!("{endpoint:?}"); - - assert!(!debug.contains("credential-value"), "{debug}"); - assert!(debug.contains("[REDACTED]"), "{debug}"); +pub(crate) fn connection_info( + endpoint: &Endpoint, + credentials: &Credentials, +) -> Result { + let (info, tls) = match endpoint { + Endpoint::Url { url, tls } => (url.expose().into_connection_info()?, Some(tls)), + Endpoint::Host(host) => { + let info = addr(&host.host, host.port, host.tls.as_ref()).into_connection_info()?; + let settings = info.redis_settings().clone().set_db(host.db); + (info.set_redis_settings(settings), host.tls.as_ref()) + } + Endpoint::Cluster(_) | Endpoint::Sentinel(_) => { + unreachable!("cluster and sentinel endpoints build their own connection info") + } + }; + let info = with_static_credentials(info, credentials) + .set_tcp_settings(TcpSettings::default().set_nodelay(true)); + match (tls, info.addr()) { + (Some(tls), ConnectionAddr::TcpTls { .. }) => apply_tls(tls, info), + _ => Ok(info), } } + +fn with_static_credentials(info: ConnectionInfo, credentials: &Credentials) -> ConnectionInfo { + let Credentials { username, password } = credentials; + let settings = info.redis_settings().clone(); + let settings = match (username, settings.username()) { + (Some(username), None) => settings.set_username(username.expose()), + _ => settings, + }; + let settings: RedisConnectionInfo = match (password, settings.password()) { + (Some(password), None) => settings.set_password(password.expose()), + _ => settings, + }; + info.set_redis_settings(settings) +} diff --git a/litellm-rust/crates/redis-client/src/connection.rs b/litellm-rust/crates/redis-client/src/connection.rs index 68170757610..1292ae5beb0 100644 --- a/litellm-rust/crates/redis-client/src/connection.rs +++ b/litellm-rust/crates/redis-client/src/connection.rs @@ -1,12 +1,26 @@ +use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + use redis::{ - Cmd, ConnectionInfo, Pipeline, RedisFuture, RedisResult, Value, - aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig}, + AsyncConnectionConfig, Cmd, ConnectionAddr, ConnectionInfo, ErrorKind, Pipeline, RedisError, + RedisFuture, RedisResult, ServerErrorKind, Value, + aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig, MultiplexedConnection}, cluster::{ClusterClientBuilder, ClusterConfig}, cluster_async::ClusterConnection, io::tcp::TcpSettings, + sentinel::{SentinelClient, SentinelClientBuilder, SentinelServerType}, }; -use crate::{ClusterEndpoint, Timeouts, error::Deployment}; +use crate::{ + ClusterEndpoint, ConnectError, Credentials, Endpoint, SentinelEndpoint, Timeouts, + config::{self, addr}, + error::Deployment, +}; // redis-py 5.3.1 redis/asyncio/connection.py: without retry options a connection gets Retry(NoBackoff(), 0) const RECONNECT_RETRIES: usize = 0; @@ -20,28 +34,51 @@ const CLUSTER_RETRY_WAIT_MS: u64 = 250; pub(crate) enum Connection { Standalone(ConnectionManager), Cluster(ClusterConnection), + Sentinel(SentinelConnection), } impl Connection { + pub(crate) fn new( + endpoint: &Endpoint, + credentials: &Credentials, + timeouts: Timeouts, + ) -> Result { + match endpoint { + Endpoint::Url { .. } | Endpoint::Host(_) => { + Self::standalone(config::connection_info(endpoint, credentials)?, timeouts) + } + Endpoint::Cluster(cluster) => Self::cluster(cluster, credentials, timeouts), + Endpoint::Sentinel(sentinel) => Ok(Self::Sentinel(SentinelConnection::new( + sentinel, + credentials, + timeouts, + )?)), + } + } + pub(crate) fn deployment(&self) -> Deployment { match self { Self::Standalone(_) => Deployment::Standalone, Self::Cluster(_) => Deployment::Cluster, + Self::Sentinel(_) => Deployment::Sentinel, } } - pub(crate) fn standalone(info: ConnectionInfo, timeouts: Timeouts) -> RedisResult { + fn standalone(info: ConnectionInfo, timeouts: Timeouts) -> Result { let config = ConnectionManagerConfig::new() .set_connection_timeout(Some(timeouts.connect)) .set_response_timeout(Some(timeouts.response)) .set_number_of_retries(RECONNECT_RETRIES); - let info = info.set_tcp_settings(TcpSettings::default().set_nodelay(true)); let manager = ConnectionManager::new_lazy_with_config(redis::Client::open(info)?, config)?; Ok(Self::Standalone(manager)) } - pub(crate) fn cluster(endpoint: &ClusterEndpoint, timeouts: Timeouts) -> RedisResult { - let builder = ClusterClientBuilder::new(endpoint.addrs()) + fn cluster( + endpoint: &ClusterEndpoint, + credentials: &Credentials, + timeouts: Timeouts, + ) -> Result { + let builder = ClusterClientBuilder::new(config::cluster_addrs(endpoint)) .tcp_settings( TcpSettings::default() .set_nodelay(true) @@ -53,11 +90,19 @@ impl Connection { .retry_wait_formula(0, 2) .min_retry_wait(CLUSTER_RETRY_WAIT_MS) .max_retry_wait(CLUSTER_RETRY_WAIT_MS + 1); - let builder = match &endpoint.username { + let builder = match &endpoint.tls { + Some(tls) => builder + .tls(config::tls_mode(tls)) + .certs(config::tls_certificates(tls)?) + .danger_accept_invalid_hostnames(!tls.check_hostname), + None => builder, + }; + let Credentials { username, password } = credentials; + let builder = match username { Some(username) => builder.username(username.expose()), None => builder, }; - let builder = match &endpoint.password { + let builder = match password { Some(password) => builder.password(password.expose()), None => builder, }; @@ -77,6 +122,7 @@ impl ConnectionLike for Connection { match self { Self::Standalone(connection) => connection.req_packed_command(cmd), Self::Cluster(connection) => connection.req_packed_command(cmd), + Self::Sentinel(connection) => connection.req_packed_command(cmd), } } @@ -89,6 +135,7 @@ impl ConnectionLike for Connection { match self { Self::Standalone(connection) => connection.req_packed_commands(pipeline, offset, count), Self::Cluster(connection) => connection.req_packed_commands(pipeline, offset, count), + Self::Sentinel(connection) => connection.req_packed_commands(pipeline, offset, count), } } @@ -96,6 +143,246 @@ impl ConnectionLike for Connection { match self { Self::Standalone(connection) => connection.get_db(), Self::Cluster(connection) => connection.get_db(), + Self::Sentinel(connection) => connection.get_db(), } } } + +#[derive(Clone)] +pub(crate) struct SentinelConnection(Arc); + +struct SentinelState { + discovery: tokio::sync::Mutex, + current: Mutex>, + generations: AtomicU64, + endpoint: SentinelEndpoint, + credentials: Credentials, + config: AsyncConnectionConfig, + connect_timeout: Duration, +} + +impl SentinelConnection { + fn new( + endpoint: &SentinelEndpoint, + credentials: &Credentials, + timeouts: Timeouts, + ) -> Result { + let sentinels = endpoint + .sentinels + .iter() + .map(|node| addr(&node.host, node.port, endpoint.tls.as_ref())); + let builder = SentinelClientBuilder::new( + sentinels, + &endpoint.service_name, + SentinelServerType::Master, + )?; + let builder = match (&endpoint.sentinel_password, &credentials.username) { + (Some(password), Some(username)) => builder + .set_client_to_sentinel_username(username.expose()) + .set_client_to_sentinel_password(password.expose()), + (Some(password), _) => builder.set_client_to_sentinel_password(password.expose()), + (None, _) => builder, + }; + let builder = match &endpoint.tls { + Some(tls) => builder + .set_client_to_sentinel_tls_mode(config::tls_mode(tls)) + .set_client_to_sentinel_certificates(config::tls_certificates(tls)?), + None => builder, + }; + let config = AsyncConnectionConfig::new() + .set_connection_timeout(Some(timeouts.connect)) + .set_response_timeout(Some(timeouts.response)); + Ok(Self(Arc::new(SentinelState { + discovery: tokio::sync::Mutex::new(builder.build()?), + current: Mutex::new(None), + generations: AtomicU64::new(0), + endpoint: endpoint.clone(), + credentials: credentials.clone(), + config, + connect_timeout: timeouts.connect, + }))) + } + + fn cached(&self) -> Option<(u64, MultiplexedConnection)> { + self.0 + .current + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + } + + async fn connection(&self) -> RedisResult<(u64, MultiplexedConnection)> { + if let Some(current) = self.cached() { + return Ok(current); + } + let mut discovery = self.0.discovery.lock().await; + if let Some(current) = self.cached() { + return Ok(current); + } + let master = tokio::time::timeout(self.0.connect_timeout, discovery.async_get_client()) + .await + .map_err(|_| timed_out("asking the Sentinels for the master timed out"))??; + let connection = self + .master_client(master.get_connection_info().addr())? + .get_multiplexed_async_connection_with_config(&self.0.config) + .await?; + let mut current = self + .0 + .current + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let generation = self.0.generations.fetch_add(1, Ordering::Relaxed); + *current = Some((generation, connection.clone())); + Ok((generation, connection)) + } + + fn master_client(&self, master: &ConnectionAddr) -> Result { + let (host, port) = match master { + ConnectionAddr::Tcp(host, port) | ConnectionAddr::TcpTls { host, port, .. } => { + (host.as_str(), *port) + } + _ => { + return Err(RedisError::from(( + ErrorKind::InvalidClientConfig, + "Sentinel named a master that is not a TCP address", + ))); + } + }; + let endpoint = Endpoint::Host(crate::HostEndpoint { + host: host.to_owned(), + port, + tls: self.0.endpoint.tls.clone(), + db: self.0.endpoint.db, + }); + let info = + config::connection_info(&endpoint, &self.0.credentials).map_err( + |error| match error { + ConnectError::Redis(error) => error, + ConnectError::TlsFile { path, .. } => RedisError::from(( + ErrorKind::InvalidClientConfig, + "reading a Redis TLS file", + path, + )), + }, + )?; + redis::Client::open(info) + } + + fn forget(&self, generation: u64) { + let mut current = self + .0 + .current + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if current + .as_ref() + .is_some_and(|(cached, _)| *cached == generation) + { + *current = None; + } + } +} + +fn timed_out(message: &'static str) -> RedisError { + RedisError::from(std::io::Error::new(std::io::ErrorKind::TimedOut, message)) +} + +fn needs_rediscovery(error: &RedisError) -> bool { + matches!( + crate::Error::classify(error.clone(), Deployment::Sentinel), + crate::Error::Transport(_) + ) +} + +fn replied_read_only(values: &[Value]) -> bool { + values.iter().any(|value| { + matches!(value, Value::ServerError(error) if error.kind() == Some(ServerErrorKind::ReadOnly)) + }) +} + +impl ConnectionLike for SentinelConnection { + fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> { + Box::pin(async move { + let (generation, mut connection) = self.connection().await?; + let reply = connection.req_packed_command(cmd).await; + if reply.as_ref().is_err_and(needs_rediscovery) { + self.forget(generation); + } + reply + }) + } + + fn req_packed_commands<'a>( + &'a mut self, + pipeline: &'a Pipeline, + offset: usize, + count: usize, + ) -> RedisFuture<'a, Vec> { + Box::pin(async move { + let (generation, mut connection) = self.connection().await?; + let replies = connection + .req_packed_commands(pipeline, offset, count) + .await; + let rediscover = match &replies { + Ok(values) => replied_read_only(values), + Err(error) => needs_rediscovery(error), + }; + if rediscover { + self.forget(generation); + } + replies + }) + } + + fn get_db(&self) -> i64 { + self.0.endpoint.db + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ClusterNode; + use litellm_redis_testing::{RedisSentinel, SERVICE_NAME}; + + #[rstest::rstest] + #[tokio::test] + async fn a_late_failure_does_not_discard_the_reconnected_sentinel_master() { + let server = RedisSentinel::start().await.unwrap(); + let (host, port) = server.sentinel(); + let endpoint = SentinelEndpoint { + sentinels: vec![ClusterNode { + host: host.to_owned(), + port, + }], + service_name: SERVICE_NAME.to_owned(), + sentinel_password: None, + tls: None, + db: 0, + }; + let mut client = + SentinelConnection::new(&endpoint, &Credentials::NONE, Timeouts::default()).unwrap(); + let (old_generation, _) = client.connection().await.unwrap(); + let original: i64 = redis::cmd("CLIENT") + .arg("ID") + .query_async(&mut client) + .await + .unwrap(); + client.forget(old_generation); + let replacement: i64 = redis::cmd("CLIENT") + .arg("ID") + .query_async(&mut client) + .await + .unwrap(); + assert_ne!(original, replacement); + + client.forget(old_generation); + let after_late_failure: i64 = redis::cmd("CLIENT") + .arg("ID") + .query_async(&mut client) + .await + .unwrap(); + + assert_eq!(replacement, after_late_failure); + } +} diff --git a/litellm-rust/crates/redis-client/src/error.rs b/litellm-rust/crates/redis-client/src/error.rs index 683832bc078..81169498926 100644 --- a/litellm-rust/crates/redis-client/src/error.rs +++ b/litellm-rust/crates/redis-client/src/error.rs @@ -13,13 +13,22 @@ pub enum Error { } #[derive(Debug, thiserror::Error)] -#[error("connecting to Redis")] -pub struct ConnectError(#[from] RedisError); +pub enum ConnectError { + #[error("connecting to Redis")] + Redis(#[from] RedisError), + #[error("reading the Redis TLS file {path}")] + TlsFile { + path: String, + #[source] + source: std::io::Error, + }, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Deployment { Standalone, Cluster, + Sentinel, } // redis-py 5.3.1 redis/_parsers/base.py EXCEPTION_CLASSES["ERR"] entries that subclass ConnectionError @@ -63,6 +72,7 @@ fn is_unavailable(kind: ServerErrorKind, deployment: Deployment) -> bool { ServerErrorKind::ClusterDown | ServerErrorKind::MasterDown, Deployment::Cluster ) + | (ServerErrorKind::ReadOnly, Deployment::Sentinel) ) } @@ -100,7 +110,8 @@ mod tests { #[case::max_clients_with_suffix("-ERR max number of clients reached, try later\r\n")] fn rejections_are_server_errors_on_every_deployment( #[case] resp: &str, - #[values(Deployment::Standalone, Deployment::Cluster)] deployment: Deployment, + #[values(Deployment::Standalone, Deployment::Cluster, Deployment::Sentinel)] + deployment: Deployment, ) { assert!(!is_transport(resp, deployment)); } @@ -117,7 +128,8 @@ mod tests { )] fn unavailable_nodes_are_transport_errors_on_every_deployment( #[case] resp: &str, - #[values(Deployment::Standalone, Deployment::Cluster)] deployment: Deployment, + #[values(Deployment::Standalone, Deployment::Cluster, Deployment::Sentinel)] + deployment: Deployment, ) { assert!(is_transport(resp, deployment)); } @@ -153,6 +165,21 @@ mod tests { Deployment::Cluster, false )] + #[case::master_down_on_sentinel( + "-MASTERDOWN Link with MASTER is down\r\n", + Deployment::Sentinel, + false + )] + #[case::cluster_down_on_sentinel( + "-CLUSTERDOWN The cluster is down\r\n", + Deployment::Sentinel, + false + )] + #[case::read_only_on_sentinel( + "-READONLY You can't write against a read only replica.\r\n", + Deployment::Sentinel, + true + )] fn topology_decides_like_redis_py( #[case] resp: &str, #[case] deployment: Deployment, diff --git a/litellm-rust/crates/redis-client/src/lib.rs b/litellm-rust/crates/redis-client/src/lib.rs index 3f4aaf678c1..d4a2385a143 100644 --- a/litellm-rust/crates/redis-client/src/lib.rs +++ b/litellm-rust/crates/redis-client/src/lib.rs @@ -10,11 +10,12 @@ mod script; pub use client::{RedisClient, Ttl}; pub use commands::Expiry; -pub use config::{ - ClusterEndpoint, ClusterNode, DEFAULT_TIMEOUT, Endpoint, HostEndpoint, RedisConfig, Timeouts, -}; +pub use config::{Credentials, DEFAULT_TIMEOUT, RedisConfig, Timeouts}; pub use error::{ConnectError, Error}; pub use events::{CallEvent, FailureKind, Operation, Outcome, RedisEvents}; +pub use litellm_redis_types::{ + ClusterEndpoint, ClusterNode, Endpoint, HostEndpoint, SentinelEndpoint, TlsOptions, +}; pub use namespace::Namespace; pub use pipeline::Pipeline; pub use redis::{ErrorKind as RedisErrorKind, RedisError, Value}; diff --git a/litellm-rust/crates/redis-client/src/pipeline.rs b/litellm-rust/crates/redis-client/src/pipeline.rs index 32457ccbcf9..666da785528 100644 --- a/litellm-rust/crates/redis-client/src/pipeline.rs +++ b/litellm-rust/crates/redis-client/src/pipeline.rs @@ -122,7 +122,9 @@ impl Pipeline { fn groups(&self, connection: &Connection) -> Vec>> { match connection { _ if self.ops.is_empty() => Vec::new(), - Connection::Standalone(_) => vec![vec![(0..self.ops.len()).collect()]], + Connection::Standalone(_) | Connection::Sentinel(_) => { + vec![vec![(0..self.ops.len()).collect()]] + } Connection::Cluster(_) => { let mut by_slot: HashMap> = HashMap::new(); for index in 0..self.ops.len() { diff --git a/litellm-rust/crates/redis-client/tests/client.rs b/litellm-rust/crates/redis-client/tests/client.rs index 972791124a3..5ba20002aed 100644 --- a/litellm-rust/crates/redis-client/tests/client.rs +++ b/litellm-rust/crates/redis-client/tests/client.rs @@ -11,8 +11,9 @@ use rstest::rstest; use litellm_circuit_breaker::{Breaker, Config, State}; use litellm_redis_client::{ - CallEvent, DEFAULT_TIMEOUT, Endpoint, Error, Expiry, FailureKind, HostEndpoint, Namespace, - Operation, Outcome, RedisClient, RedisConfig, RedisEvents, Script, Timeouts, Ttl, + CallEvent, Credentials, DEFAULT_TIMEOUT, Endpoint, Error, Expiry, FailureKind, HostEndpoint, + Namespace, Operation, Outcome, RedisClient, RedisConfig, RedisEvents, Script, Timeouts, + TlsOptions, Ttl, }; use litellm_secret_value::SecretValue; use support::{KEEP, connect, connect_guarded, connections, endpoint}; @@ -252,6 +253,7 @@ async fn outage_events_report_the_failure_kind( timeouts: Timeouts::new(Duration::from_millis(300), None), breaker: None, events: Some(recorded.clone()), + credentials: Credentials::NONE, }) .unwrap(); client.ping().await.unwrap(); @@ -301,6 +303,7 @@ async fn every_call_reports_its_operation_and_outcome( let recorded = Arc::new(Recorded::default()); let client = RedisClient::new(RedisConfig { endpoint: endpoint(&server), + credentials: Credentials::NONE, namespace: Namespace::default(), timeouts: Timeouts::default(), breaker: None, @@ -729,19 +732,21 @@ async fn a_host_endpoint_connects_to_its_database() { let connect = |endpoint| { RedisClient::new(RedisConfig { endpoint, + credentials: Credentials::NONE, namespace: Namespace::default(), timeouts: Timeouts::default(), breaker: None, events: None, }) }; - let url = |url: String| Endpoint::Url(SecretValue::new(url)); + let url = |url: String| Endpoint::Url { + url: SecretValue::new(url), + tls: TlsOptions::verified(), + }; let in_db_3 = connect(Endpoint::Host(HostEndpoint { host: "127.0.0.1".to_owned(), port, - username: None, - password: None, - tls: false, + tls: None, db: 3, })) .unwrap(); diff --git a/litellm-rust/crates/redis-client/tests/config.rs b/litellm-rust/crates/redis-client/tests/config.rs new file mode 100644 index 00000000000..206cd906e05 --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/config.rs @@ -0,0 +1,142 @@ +use rstest::rstest; + +use litellm_redis_client::{Credentials, Endpoint, SentinelEndpoint, TlsOptions}; +use litellm_secret_value::SecretValue; + +#[rstest] +#[case::url(format!("{:?}", Endpoint::Url { url: SecretValue::new("redis://user:credential-value@cache:6379/0"), tls: TlsOptions::verified() }))] +#[case::static_password(format!("{:?}", Credentials { username: None, password: Some(SecretValue::new("credential-value")) }))] +#[case::sentinel_password(format!("{:?}", Endpoint::Sentinel(SentinelEndpoint { + sentinels: Vec::new(), + service_name: "primary".to_owned(), + sentinel_password: Some(SecretValue::new("credential-value")), + tls: None, + db: 0, +})))] +fn debug_output_never_shows_the_secret(#[case] debug: String) { + assert!(!debug.contains("credential-value"), "{debug}"); + assert!(debug.contains("[REDACTED]"), "{debug}"); +} + +#[derive(Clone, Copy, Debug)] +enum Ca { + Trusted, + SystemRoots, +} + +async fn ping_over_tls( + require_client_certificate: bool, + host: &str, + ca: Ca, + verify_certificate: bool, + check_hostname: bool, + client_certificate: bool, +) -> Result<(), litellm_redis_client::Error> { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } + let server = litellm_redis_testing::RedisTlsServer::start(require_client_certificate) + .await + .unwrap(); + let files = server.files(); + let tls = TlsOptions { + ca_certs: matches!(ca, Ca::Trusted).then(|| files.ca.clone()), + certfile: client_certificate.then(|| files.client_cert.clone()), + keyfile: client_certificate.then(|| files.client_key.clone()), + verify_certificate, + check_hostname, + }; + let client = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { + endpoint: Endpoint::Host(litellm_redis_client::HostEndpoint { + host: host.to_owned(), + port: server.port(), + tls: Some(tls), + db: 0, + }), + credentials: Credentials::NONE, + namespace: litellm_redis_client::Namespace::default(), + timeouts: litellm_redis_client::Timeouts::default(), + breaker: None, + events: None, + }) + .unwrap(); + client.ping().await +} + +#[rstest] +#[case::trusted_ca_without_hostname_check_like_redis_py( + "127.0.0.1", + Ca::Trusted, + true, + false, + true +)] +#[case::hostname_check_with_a_matching_name("localhost", Ca::Trusted, true, true, true)] +#[case::hostname_check_against_an_unlisted_address("127.0.0.1", Ca::Trusted, true, true, false)] +#[case::an_untrusted_ca_is_refused("127.0.0.1", Ca::SystemRoots, true, false, false)] +#[case::cert_reqs_none_skips_verification("127.0.0.1", Ca::SystemRoots, false, false, true)] +#[tokio::test] +async fn tls_options_decide_which_servers_are_trusted( + #[case] host: &str, + #[case] ca: Ca, + #[case] verify_certificate: bool, + #[case] check_hostname: bool, + #[case] connects: bool, +) { + let result = ping_over_tls(false, host, ca, verify_certificate, check_hostname, false).await; + + assert_eq!(result.is_ok(), connects, "{result:?}"); + if let Err(error) = result { + assert!( + matches!(error, litellm_redis_client::Error::Transport(_)), + "{error:?}" + ); + } +} + +#[rstest] +#[case::with_the_client_certificate(true, true)] +#[case::without_a_client_certificate(false, false)] +#[tokio::test] +async fn mutual_tls_sends_the_configured_client_certificate( + #[case] client_certificate: bool, + #[case] connects: bool, +) { + let result = ping_over_tls( + true, + "127.0.0.1", + Ca::Trusted, + true, + false, + client_certificate, + ) + .await; + + assert_eq!(result.is_ok(), connects, "{result:?}"); +} + +#[rstest] +fn a_missing_tls_file_names_the_path() { + let tls = TlsOptions { + ca_certs: Some("/nonexistent/ca.pem".into()), + ..TlsOptions::verified() + }; + + let error = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { + endpoint: Endpoint::Host(litellm_redis_client::HostEndpoint { + host: "127.0.0.1".to_owned(), + port: 6379, + tls: Some(tls), + db: 0, + }), + credentials: Credentials::NONE, + namespace: litellm_redis_client::Namespace::default(), + timeouts: litellm_redis_client::Timeouts::default(), + breaker: None, + events: None, + }) + .err() + .unwrap(); + + assert!(error.to_string().contains("/nonexistent/ca.pem"), "{error}"); +} diff --git a/litellm-rust/crates/redis-client/tests/connection.rs b/litellm-rust/crates/redis-client/tests/connection.rs index 44a7ded9335..c8e414a4856 100644 --- a/litellm-rust/crates/redis-client/tests/connection.rs +++ b/litellm-rust/crates/redis-client/tests/connection.rs @@ -2,57 +2,56 @@ mod support; use std::time::Duration; -use litellm_redis_client::{Endpoint, Error, HostEndpoint}; -use litellm_redis_testing::RedisServer; -use litellm_secret_value::SecretValue; +use litellm_redis_testing::{ + RedisDeployment, RedisSentinel, RedisServer, SERVICE_NAME, Topology, eventually, +}; use rstest::rstest; -use support::connect_to; +use litellm_redis_client::{ + ClusterNode, Credentials, Endpoint, Error, HostEndpoint, Namespace, RedisClient, RedisConfig, + SentinelEndpoint, Timeouts, +}; +use litellm_secret_value::SecretValue; +use support::connect_as; #[rstest] #[tokio::test] async fn an_out_of_range_database_is_a_server_rejection() { let server = RedisServer::start().await.unwrap(); let port = server.url().rsplit(':').next().unwrap().parse().unwrap(); - let client = connect_to( - Endpoint::Host(HostEndpoint { + let client = RedisClient::new(RedisConfig { + endpoint: Endpoint::Host(HostEndpoint { host: "127.0.0.1".into(), port, - username: None, - password: None, - tls: false, + tls: None, db: 999, }), - "", - Duration::from_secs(2), - None, - ); + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_secs(2), None), + breaker: None, + events: None, + }) + .unwrap(); let result = client.ping().await; assert!(matches!(result, Err(Error::Server(_))), "{result:?}"); } #[rstest] #[tokio::test] -async fn a_rejected_password_is_a_transport_failure() { - let server = RedisServer::start().await.unwrap(); +async fn a_rejected_credential_is_a_transport_failure() { + let server = RedisDeployment::start(Topology::Standalone).await.unwrap(); server - .cli(&["ACL", "SETUSER", "default", "on", ">right"]) + .cli_on_every_node(&["ACL", "SETUSER", "default", "on", ">right"]) .await .unwrap(); - let port = server.url().rsplit(':').next().unwrap().parse().unwrap(); let with_password = |password: &str| { - connect_to( - Endpoint::Host(HostEndpoint { - host: "127.0.0.1".to_owned(), - port, + connect_as( + &server, + Credentials { username: None, password: Some(SecretValue::new(password)), - tls: false, - db: 0, - }), - "", - Duration::from_secs(2), - None, + }, ) }; with_password("right").ping().await.unwrap(); @@ -61,3 +60,71 @@ async fn a_rejected_password_is_a_transport_failure() { assert!(matches!(rejected, Err(Error::Transport(_))), "{rejected:?}"); } + +fn sentinel_client(sentinel: &RedisSentinel) -> RedisClient { + let (host, port) = sentinel.sentinel(); + RedisClient::new(RedisConfig { + endpoint: Endpoint::Sentinel(SentinelEndpoint { + sentinels: vec![ClusterNode { + host: host.to_owned(), + port, + }], + service_name: SERVICE_NAME.to_owned(), + sentinel_password: None, + tls: None, + db: 0, + }), + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_secs(1), None), + breaker: None, + events: None, + }) + .unwrap() +} + +#[rstest] +#[tokio::test] +async fn sentinel_writes_follow_the_master_through_a_failover() { + let sentinel = RedisSentinel::start().await.unwrap(); + let client = sentinel_client(&sentinel); + client + .set("before", "1", Duration::from_secs(600)) + .await + .unwrap(); + eventually(async || { + let replica = + redis::Client::open(format!("redis://127.0.0.1:{}", sentinel.replica_port())).ok()?; + let mut replica = replica.get_multiplexed_async_connection().await.ok()?; + let value: Option = redis::cmd("GET") + .arg("before") + .query_async(&mut replica) + .await + .ok()?; + value + }) + .await; + + let promoted = sentinel.failover().await.unwrap(); + eventually(async || { + client + .set("after", "2", Duration::from_secs(600)) + .await + .ok() + }) + .await; + + assert_eq!(promoted, sentinel.replica_port()); + assert_eq!( + client.get::("before").await.unwrap().as_deref(), + Some("1") + ); + let promoted = redis::Client::open(format!("redis://127.0.0.1:{promoted}")).unwrap(); + let mut promoted = promoted.get_multiplexed_async_connection().await.unwrap(); + let after: Option = redis::cmd("GET") + .arg("after") + .query_async(&mut promoted) + .await + .unwrap(); + assert_eq!(after.as_deref(), Some("2")); +} diff --git a/litellm-rust/crates/redis-client/tests/support/mod.rs b/litellm-rust/crates/redis-client/tests/support/mod.rs index a8b561db628..258fdcc9bc3 100644 --- a/litellm-rust/crates/redis-client/tests/support/mod.rs +++ b/litellm-rust/crates/redis-client/tests/support/mod.rs @@ -4,7 +4,8 @@ use std::time::Duration; use litellm_circuit_breaker::Breaker; use litellm_redis_client::{ - ClusterEndpoint, ClusterNode, Endpoint, Expiry, Namespace, RedisClient, RedisConfig, Timeouts, + ClusterEndpoint, ClusterNode, Credentials, Endpoint, Expiry, Namespace, RedisClient, + RedisConfig, Timeouts, TlsOptions, }; use litellm_redis_testing::{ClientConnection, RedisDeployment}; use litellm_secret_value::SecretValue; @@ -13,7 +14,10 @@ pub const KEEP: Expiry = Expiry::KeepExisting(Duration::from_secs(60)); pub fn endpoint(deployment: &RedisDeployment) -> Endpoint { match deployment { - RedisDeployment::Standalone(server) => Endpoint::Url(SecretValue::new(server.url())), + RedisDeployment::Standalone(server) => Endpoint::Url { + url: SecretValue::new(server.url()), + tls: TlsOptions::verified(), + }, RedisDeployment::Cluster(cluster) => Endpoint::Cluster(ClusterEndpoint { nodes: cluster .nodes() @@ -22,9 +26,7 @@ pub fn endpoint(deployment: &RedisDeployment) -> Endpoint { port, }) .collect(), - username: None, - password: None, - tls: false, + tls: None, }), } } @@ -38,18 +40,10 @@ pub fn connect_guarded( namespace: &str, timeout: Duration, breaker: Option, -) -> RedisClient { - connect_to(endpoint(deployment), namespace, timeout, breaker) -} - -pub fn connect_to( - endpoint: Endpoint, - namespace: &str, - timeout: Duration, - breaker: Option, ) -> RedisClient { RedisClient::new(RedisConfig { - endpoint, + endpoint: endpoint(deployment), + credentials: Credentials::NONE, namespace: Namespace::new(namespace), timeouts: Timeouts::new(timeout, None), breaker, @@ -58,6 +52,18 @@ pub fn connect_to( .unwrap() } +pub fn connect_as(deployment: &RedisDeployment, credentials: Credentials) -> RedisClient { + RedisClient::new(RedisConfig { + endpoint: endpoint(deployment), + credentials, + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_secs(2), None), + breaker: None, + events: None, + }) + .unwrap() +} + pub async fn connections(deployment: &RedisDeployment) -> Vec { deployment.clients().await.unwrap() } diff --git a/litellm-rust/crates/redis-connect/AGENTS.md b/litellm-rust/crates/redis-connect/AGENTS.md new file mode 100644 index 00000000000..cc2d8f3fb11 --- /dev/null +++ b/litellm-rust/crates/redis-connect/AGENTS.md @@ -0,0 +1,3 @@ +- Turns a `litellm_redis_types::ResolvedRedis` into a `RedisClient`, the same way for every host. It depends on `litellm-redis-types`, never on `litellm-config`, so a host that resolves Redis some other way connects the same way. `redis-client` stays cloud-agnostic, so credentials are mapped here +- `RedisAuth::Host` (a Python `credential_provider` object) fails at connect with an error naming the key, since no Rust host can build one. Cloud auth (GCP, Azure, AWS) fails the same way until its token sources land, so a config that needs it never silently runs without Redis +- Tests run against real Redis from `litellm-redis-testing` diff --git a/litellm-rust/crates/redis-connect/Cargo.toml b/litellm-rust/crates/redis-connect/Cargo.toml new file mode 100644 index 00000000000..efc1b927afe --- /dev/null +++ b/litellm-rust/crates/redis-connect/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "litellm-redis-connect" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-circuit-breaker.workspace = true +litellm-redis-types.workspace = true +litellm-redis-client.workspace = true +thiserror.workspace = true +tracing.workspace = true + +[dev-dependencies] +litellm-config.workspace = true +litellm-redis-testing.workspace = true +rustls.workspace = true +serde_json.workspace = true +rstest.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/litellm-rust/crates/redis-connect/src/connect.rs b/litellm-rust/crates/redis-connect/src/connect.rs new file mode 100644 index 00000000000..c2a3b4b6f45 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/connect.rs @@ -0,0 +1,38 @@ +use litellm_circuit_breaker::Breaker; +use litellm_redis_client::{ + Credentials, DEFAULT_TIMEOUT, Namespace, RedisClient, RedisConfig, Timeouts, +}; +use litellm_redis_types::{RedisAuth, ResolvedRedis}; + +use crate::Error; + +pub fn connect(resolved: ResolvedRedis, breaker: Option) -> Result { + tracing::info!(endpoint = ?resolved.endpoint, namespace = ?resolved.namespace, "redis client configured"); + Ok(RedisClient::new(RedisConfig { + endpoint: resolved.endpoint, + credentials: credentials(resolved.auth)?, + namespace: resolved.namespace.map(Namespace::new).unwrap_or_default(), + timeouts: Timeouts::new( + resolved.timeout.unwrap_or(DEFAULT_TIMEOUT), + resolved.connect_timeout, + ), + breaker, + events: None, + })?) +} + +fn credentials(auth: RedisAuth) -> Result { + match auth { + RedisAuth::Static { username, password } => Ok(Credentials { username, password }), + RedisAuth::Host => Err(Error::NoHostCredentialProvider), + RedisAuth::GcpIam { .. } => Err(Error::CloudAuth { + setting: "gcp_service_account", + }), + RedisAuth::AzureAd(_) => Err(Error::CloudAuth { + setting: "azure_redis_ad_token", + }), + RedisAuth::AwsIam(_) => Err(Error::CloudAuth { + setting: "aws_iam_auth", + }), + } +} diff --git a/litellm-rust/crates/redis-connect/src/error.rs b/litellm-rust/crates/redis-connect/src/error.rs new file mode 100644 index 00000000000..779dd47924b --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/error.rs @@ -0,0 +1,13 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Connect(#[from] litellm_redis_client::ConnectError), + #[error( + "Redis setting credential_provider names a Python object, which this host cannot supply" + )] + NoHostCredentialProvider, + #[error( + "Redis setting {setting} asks for cloud authentication, which this host does not support yet" + )] + CloudAuth { setting: &'static str }, +} diff --git a/litellm-rust/crates/redis-connect/src/lib.rs b/litellm-rust/crates/redis-connect/src/lib.rs new file mode 100644 index 00000000000..733c98069b6 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/lib.rs @@ -0,0 +1,5 @@ +mod connect; +mod error; + +pub use connect::connect; +pub use error::Error; diff --git a/litellm-rust/crates/redis-connect/tests/connect.rs b/litellm-rust/crates/redis-connect/tests/connect.rs new file mode 100644 index 00000000000..ba0530cb425 --- /dev/null +++ b/litellm-rust/crates/redis-connect/tests/connect.rs @@ -0,0 +1,133 @@ +use std::{collections::HashMap, time::Duration}; + +use litellm_config::{RedisSettings, SecretSnapshot}; +use litellm_redis_connect::{Error, connect}; +use litellm_redis_testing::{RedisSentinel, RedisServer, RedisTlsServer, SERVICE_NAME, eventually}; +use litellm_redis_types::ResolvedRedis; +use rstest::rstest; + +fn resolved(kwargs: serde_json::Value, environment: &[(&str, &str)]) -> ResolvedRedis { + let settings: RedisSettings = serde_json::from_value(kwargs).unwrap(); + let environment: HashMap<&str, &str> = environment.iter().copied().collect(); + let lookup = |name: &str| environment.get(name).map(|value| (*value).to_owned()); + let configured = settings.coordination().unwrap(); + configured + .resolve( + &SecretSnapshot::read(configured.secret_names(), &lookup), + &lookup, + ) + .unwrap() + .redis +} + +fn port(url: &str) -> u16 { + url.rsplit(':').next().unwrap().parse().unwrap() +} + +#[rstest] +#[case::trusted_ca(true, true)] +#[case::system_roots_only(false, false)] +#[tokio::test] +async fn python_tls_kwargs_reach_the_server(#[case] with_ca: bool, #[case] connects: bool) { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } + let server = RedisTlsServer::start(false).await.unwrap(); + let kwargs = serde_json::json!({ + "host": server.host(), + "port": server.port(), + "ssl": true, + "ssl_ca_certs": with_ca.then(|| server.files().ca.display().to_string()), + "socket_timeout": 2, + }); + + let pinged = connect(resolved(kwargs, &[]), None).unwrap().ping().await; + + assert_eq!(pinged.is_ok(), connects, "{pinged:?}"); +} + +#[rstest] +#[case::configured_password(Some("right"), &[], true)] +#[case::environment_password(None, &[("REDIS_PASSWORD", "right")], true)] +#[case::wrong_password(Some("wrong"), &[], false)] +#[tokio::test] +async fn static_credentials_from_kwargs_or_environment_authenticate( + #[case] password: Option<&str>, + #[case] environment: &[(&str, &str)], + #[case] connects: bool, +) { + let server = RedisServer::start().await.unwrap(); + server + .cli(&["CONFIG", "SET", "requirepass", "right"]) + .await + .unwrap(); + let kwargs = serde_json::json!({ + "host": "127.0.0.1", + "port": port(server.url()), + "password": password, + "socket_timeout": 2, + }); + + let pinged = connect(resolved(kwargs, environment), None) + .unwrap() + .ping() + .await; + + assert_eq!(pinged.is_ok(), connects, "{pinged:?}"); +} + +#[rstest] +#[tokio::test] +async fn python_sentinel_kwargs_follow_the_master_through_a_failover() { + let sentinel = RedisSentinel::start().await.unwrap(); + let (host, sentinel_port) = sentinel.sentinel(); + let kwargs = serde_json::json!({ + "sentinel_nodes": [[host, sentinel_port]], + "service_name": SERVICE_NAME, + "socket_timeout": 1, + }); + let client = connect(resolved(kwargs, &[]), None).unwrap(); + client + .set("before", "1", Duration::from_secs(600)) + .await + .unwrap(); + + let promoted = sentinel.failover().await.unwrap(); + eventually(async || { + client + .set("after", "2", Duration::from_secs(600)) + .await + .ok() + }) + .await; + + assert_eq!(promoted, sentinel.replica_port()); + assert_eq!(sentinel.master_port().await.unwrap(), promoted); + assert_eq!( + client.get::("after").await.unwrap().as_deref(), + Some("2") + ); +} + +#[rstest] +#[case::credential_provider(serde_json::json!({"host": "redis", "credential_provider": true}), &[], None)] +#[case::gcp(serde_json::json!({"host": "redis", "gcp_service_account": "account"}), &[], Some("gcp_service_account"))] +#[case::azure(serde_json::json!({"host": "redis"}), &[("REDIS_AZURE_AD_TOKEN", "true")], Some("azure_redis_ad_token"))] +#[case::aws( + serde_json::json!({"host": "cache", "ssl": true, "aws_iam_auth": true, "aws_iam_user_name": "user", "aws_iam_cache_name": "cache", "aws_iam_region": "us-east-1"}), + &[], + Some("aws_iam_auth") +)] +fn auth_this_host_cannot_build_fails_at_boot_naming_the_setting( + #[case] kwargs: serde_json::Value, + #[case] environment: &[(&str, &str)], + #[case] cloud_setting: Option<&str>, +) { + let error = connect(resolved(kwargs, environment), None).err().unwrap(); + + match (error, cloud_setting) { + (Error::NoHostCredentialProvider, None) => {} + (Error::CloudAuth { setting }, Some(expected)) => assert_eq!(setting, expected), + (error, _) => panic!("unexpected {error:?}"), + } +} diff --git a/litellm-rust/crates/redis-testing/AGENTS.md b/litellm-rust/crates/redis-testing/AGENTS.md index 225e4a71203..4f106caab9a 100644 --- a/litellm-rust/crates/redis-testing/AGENTS.md +++ b/litellm-rust/crates/redis-testing/AGENTS.md @@ -4,6 +4,8 @@ - `RedisCluster` runs 3 masters in one container. Each node listens on the same port inside and outside and announces 127.0.0.1, so MOVED redirects work from the test process - Outages come from the container (`restart`, `freeze` / `thaw`), never `DEBUG`, which the official image disables - No env gate that makes a test pass without Redis. A test that can't start its container fails +- `RedisTlsServer` serves TLS with certificates `rcgen` generates per start, for `localhost`, optionally requiring a client certificate +- `RedisSentinel` runs a primary, a replica and one sentinel in one container, all announcing 127.0.0.1. `failover()` returns once the replica is master and the old master is a replica - `cli` / `cli_on_every_node` run `redis-cli -e` inside the container for setup such as ACL users, so tests don't need a second client. `-e` makes an error reply fail the call, and a test whose setup changes server state also asserts that state before relying on it - Not `redis-test`'s server helpers, which need local `redis-server` and `openssl` binaries - `RedisDeployment` runs either `Topology`, so a test can take `#[values(Topology::Standalone, Topology::Cluster)]`. `clients()` returns every client connection on every node from `CLIENT LIST`, leaving out the CLI's own diff --git a/litellm-rust/crates/redis-testing/Cargo.toml b/litellm-rust/crates/redis-testing/Cargo.toml index 5afef2f1aa9..18fefb273bc 100644 --- a/litellm-rust/crates/redis-testing/Cargo.toml +++ b/litellm-rust/crates/redis-testing/Cargo.toml @@ -6,6 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +rcgen = "0.14.10" +tempfile.workspace = true testcontainers-modules = { workspace = true, features = ["redis"] } thiserror.workspace = true tokio = { workspace = true, features = ["io-util", "time"] } diff --git a/litellm-rust/crates/redis-testing/src/error.rs b/litellm-rust/crates/redis-testing/src/error.rs index 59a60f37a72..ac0a526a293 100644 --- a/litellm-rust/crates/redis-testing/src/error.rs +++ b/litellm-rust/crates/redis-testing/src/error.rs @@ -10,4 +10,10 @@ pub enum Error { Container(#[from] testcontainers_modules::testcontainers::TestcontainersError), #[error("{command} exited with {code}")] Command { command: String, code: i64 }, + #[error("generating test TLS certificates")] + Certificate(#[from] rcgen::Error), + #[error("writing test TLS files")] + Files(#[source] io::Error), + #[error("Sentinel did not promote the replica")] + Failover, } diff --git a/litellm-rust/crates/redis-testing/src/lib.rs b/litellm-rust/crates/redis-testing/src/lib.rs index 4a1365c1a28..905d387200b 100644 --- a/litellm-rust/crates/redis-testing/src/lib.rs +++ b/litellm-rust/crates/redis-testing/src/lib.rs @@ -3,9 +3,13 @@ mod deployment; mod error; mod exec; mod ports; +mod sentinel; mod standalone; +mod tls; pub use cluster::RedisCluster; pub use deployment::{ClientConnection, RedisDeployment, Topology, eventually}; pub use error::Error; +pub use sentinel::{RedisSentinel, SERVICE_NAME}; pub use standalone::RedisServer; +pub use tls::{RedisTlsServer, SERVER_NAME, TlsFiles}; diff --git a/litellm-rust/crates/redis-testing/src/ports.rs b/litellm-rust/crates/redis-testing/src/ports.rs index e7b94988c10..eb0d5bd4e2e 100644 --- a/litellm-rust/crates/redis-testing/src/ports.rs +++ b/litellm-rust/crates/redis-testing/src/ports.rs @@ -42,6 +42,17 @@ pub(crate) async fn wait_for_pong(port: u16) -> Result<(), Error> { Err(Error::Unreachable { port }) } +pub(crate) async fn wait_for_listener(port: u16) -> Result<(), Error> { + let deadline = Instant::now() + READY_TIMEOUT; + while Instant::now() < deadline { + if TcpStream::connect((HOST, port)).await.is_ok() { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(Error::Unreachable { port }) +} + async fn ping(port: u16) -> io::Result { let mut stream = TcpStream::connect((HOST, port)).await?; stream.write_all(b"PING\r\n").await?; diff --git a/litellm-rust/crates/redis-testing/src/sentinel.rs b/litellm-rust/crates/redis-testing/src/sentinel.rs new file mode 100644 index 00000000000..928a892cf61 --- /dev/null +++ b/litellm-rust/crates/redis-testing/src/sentinel.rs @@ -0,0 +1,122 @@ +use std::time::{Duration, Instant}; + +use testcontainers_modules::testcontainers::{ + ContainerAsync, GenericImage, ImageExt, + core::{IntoContainerPort, WaitFor}, + runners::AsyncRunner, +}; + +use crate::{ + Error, + exec::{cli, run}, + ports::{HOST, start_on_free_ports, wait_for_pong}, + standalone::REDIS_TAG, +}; + +pub const SERVICE_NAME: &str = "primary"; +const READY: &str = "redis-testing sentinel ready"; +const FAILOVER_TIMEOUT: Duration = Duration::from_secs(30); + +const START_SENTINEL: &str = r#" +set -e +primary=$1 replica=$2 sentinel=$3 +common="--save '' --appendonly no --daemonize yes --replica-announce-ip 127.0.0.1" +eval redis-server --port "$primary" $common +eval redis-server --port "$replica" --replicaof 127.0.0.1 "$primary" $common +until redis-cli -p "$replica" info replication | grep -q master_link_status:up; do sleep 0.1; done +cat > /data/sentinel.conf <, + ports: [u16; 3], +} + +impl RedisSentinel { + pub async fn start() -> Result { + start_on_free_ports(Self::start_on).await + } + + async fn start_on(ports: [u16; 3]) -> Result { + let image = GenericImage::new("redis", REDIS_TAG) + .with_wait_for(WaitFor::message_on_stdout(READY)) + .with_entrypoint("sh"); + let request = ports.iter().fold( + image.with_cmd( + ["-c", START_SENTINEL, "sh"] + .into_iter() + .map(str::to_owned) + .chain(ports.iter().map(u16::to_string)), + ), + |request, &port| request.with_mapped_port(port, port.tcp()), + ); + let container = request.start().await?; + for port in ports { + wait_for_pong(port).await?; + } + Ok(Self { container, ports }) + } + + pub fn sentinel(&self) -> (&'static str, u16) { + (HOST, self.ports[2]) + } + + pub fn primary_port(&self) -> u16 { + self.ports[0] + } + + pub fn replica_port(&self) -> u16 { + self.ports[1] + } + + pub async fn master_port(&self) -> Result { + let reply = run( + &self.container, + cli( + self.ports[2], + &["sentinel", "get-master-addr-by-name", SERVICE_NAME], + ), + ) + .await?; + Ok(reply + .lines() + .nth(1) + .and_then(|port| port.trim().parse().ok()) + .unwrap_or_default()) + } + + pub async fn failover(&self) -> Result { + let before = self.master_port().await?; + run( + &self.container, + cli(self.ports[2], &["sentinel", "failover", SERVICE_NAME]), + ) + .await?; + let deadline = Instant::now() + FAILOVER_TIMEOUT; + loop { + let now = self.master_port().await?; + let promoted = now != before && self.role(now).await?.starts_with("master"); + if promoted && self.role(before).await?.starts_with("slave") { + return Ok(now); + } + if Instant::now() > deadline { + return Err(Error::Failover); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + async fn role(&self, port: u16) -> Result { + run(&self.container, cli(port, &["role"])).await + } +} diff --git a/litellm-rust/crates/redis-testing/src/tls.rs b/litellm-rust/crates/redis-testing/src/tls.rs new file mode 100644 index 00000000000..65cb9b1c79b --- /dev/null +++ b/litellm-rust/crates/redis-testing/src/tls.rs @@ -0,0 +1,131 @@ +use std::path::PathBuf; + +use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, DnType, IsCa, KeyPair}; +use tempfile::TempDir; +use testcontainers_modules::testcontainers::{ + ContainerAsync, GenericImage, ImageExt, + core::{IntoContainerPort, WaitFor}, + runners::AsyncRunner, +}; + +use crate::{ + Error, + ports::{HOST, start_on_free_ports, wait_for_listener}, + standalone::REDIS_TAG, +}; + +pub const SERVER_NAME: &str = "localhost"; + +pub struct TlsFiles { + _directory: TempDir, + pub ca: PathBuf, + pub client_cert: PathBuf, + pub client_key: PathBuf, +} + +struct Generated { + files: TlsFiles, + ca_pem: String, + server_cert: String, + server_key: String, +} + +pub struct RedisTlsServer { + _container: ContainerAsync, + port: u16, + files: TlsFiles, +} + +impl RedisTlsServer { + pub async fn start(require_client_certificate: bool) -> Result { + start_on_free_ports(|[port]| Self::start_on(port, require_client_certificate)).await + } + + async fn start_on(port: u16, require_client_certificate: bool) -> Result { + let generated = generate()?; + let auth_clients = match require_client_certificate { + true => "yes", + false => "no", + }; + let container = GenericImage::new("redis", REDIS_TAG) + .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections")) + .with_copy_to("/tls/ca.crt", generated.ca_pem.into_bytes()) + .with_copy_to("/tls/server.crt", generated.server_cert.into_bytes()) + .with_copy_to("/tls/server.key", generated.server_key.into_bytes()) + .with_cmd([ + "redis-server", + "--port", + "0", + "--tls-port", + &port.to_string(), + "--tls-cert-file", + "/tls/server.crt", + "--tls-key-file", + "/tls/server.key", + "--tls-ca-cert-file", + "/tls/ca.crt", + "--tls-auth-clients", + auth_clients, + ]) + .with_mapped_port(port, port.tcp()) + .start() + .await?; + wait_for_listener(port).await?; + Ok(Self { + _container: container, + port, + files: generated.files, + }) + } + + pub fn host(&self) -> &'static str { + HOST + } + + pub fn port(&self) -> u16 { + self.port + } + + pub fn files(&self) -> &TlsFiles { + &self.files + } +} + +fn generate() -> Result { + let named = |names: Vec, common_name: &str| -> Result { + let mut params = CertificateParams::new(names)?; + params + .distinguished_name + .push(DnType::CommonName, common_name); + Ok(params) + }; + let mut ca_params = named(Vec::new(), "redis-testing CA")?; + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca = CertifiedIssuer::self_signed(ca_params, KeyPair::generate()?)?; + let server_key = KeyPair::generate()?; + let server = + named(vec![SERVER_NAME.to_owned()], "redis-testing server")?.signed_by(&server_key, &ca)?; + let client_key = KeyPair::generate()?; + let client = + named(vec!["client".to_owned()], "redis-testing client")?.signed_by(&client_key, &ca)?; + let directory = tempfile::tempdir().map_err(Error::Files)?; + let write = |name: &str, contents: String| -> Result { + let path = directory.path().join(name); + std::fs::write(&path, contents).map_err(Error::Files)?; + Ok(path) + }; + let ca_path = write("ca.pem", ca.pem())?; + let client_cert = write("client.pem", client.pem())?; + let client_key_path = write("client.key", client_key.serialize_pem())?; + Ok(Generated { + ca_pem: ca.pem(), + server_cert: server.pem(), + server_key: server_key.serialize_pem(), + files: TlsFiles { + _directory: directory, + ca: ca_path, + client_cert, + client_key: client_key_path, + }, + }) +} diff --git a/litellm-rust/crates/redis-types/AGENTS.md b/litellm-rust/crates/redis-types/AGENTS.md new file mode 100644 index 00000000000..c9d18fe3da1 --- /dev/null +++ b/litellm-rust/crates/redis-types/AGENTS.md @@ -0,0 +1,2 @@ +- `ResolvedRedis` and the endpoint, node, TLS and auth types inside it: what `config` resolves to, `redis-connect` turns into a client and `redis-client` connects with, so none of them converts another's copy +- Data only. It depends on `secret-value` and nothing else, so `config` never pulls in redis-rs or tokio. Anything that needs redis-rs, like building a `ConnectionInfo` or reading TLS files, lives in `redis-client` diff --git a/litellm-rust/crates/redis-types/Cargo.toml b/litellm-rust/crates/redis-types/Cargo.toml new file mode 100644 index 00000000000..229325a27bd --- /dev/null +++ b/litellm-rust/crates/redis-types/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "litellm-redis-types" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-secret-value.workspace = true diff --git a/litellm-rust/crates/redis-types/src/lib.rs b/litellm-rust/crates/redis-types/src/lib.rs new file mode 100644 index 00000000000..7b1365d50b9 --- /dev/null +++ b/litellm-rust/crates/redis-types/src/lib.rs @@ -0,0 +1,100 @@ +use std::{path::PathBuf, time::Duration}; + +use litellm_secret_value::SecretValue; + +/// One Redis connection with every setting, environment value and secret already applied, the +/// value a host hands to the connector. +#[derive(Clone, Debug, PartialEq)] +pub struct ResolvedRedis { + pub endpoint: Endpoint, + pub auth: RedisAuth, + pub namespace: Option, + pub timeout: Option, + pub connect_timeout: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Endpoint { + Url { url: SecretValue, tls: TlsOptions }, + Host(HostEndpoint), + Cluster(ClusterEndpoint), + Sentinel(SentinelEndpoint), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct HostEndpoint { + pub host: String, + pub port: u16, + pub tls: Option, + pub db: i64, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ClusterEndpoint { + pub nodes: Vec, + pub tls: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ClusterNode { + pub host: String, + pub port: u16, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SentinelEndpoint { + pub sentinels: Vec, + pub service_name: String, + pub sentinel_password: Option, + pub tls: Option, + pub db: i64, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TlsOptions { + pub ca_certs: Option, + pub certfile: Option, + pub keyfile: Option, + pub verify_certificate: bool, + pub check_hostname: bool, +} + +impl TlsOptions { + pub fn verified() -> Self { + Self { + verify_certificate: true, + ..Self::default() + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum RedisAuth { + Static { + username: Option, + password: Option, + }, + /// The host supplies credentials, like a redis-py `credential_provider` object. + Host, + GcpIam { + service_account: String, + }, + AzureAd(AzureAdAuth), + AwsIam(AwsIamAuth), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AzureAdAuth { + pub client_id: Option, + pub tenant_id: Option, + pub client_secret: Option, + pub username: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AwsIamAuth { + pub user_name: String, + pub cache_name: String, + pub region: String, + pub serverless: bool, +} diff --git a/tests/unit/test_redis.py b/tests/unit/test_redis.py index bfe3b55497d..6e2c0c31165 100644 --- a/tests/unit/test_redis.py +++ b/tests/unit/test_redis.py @@ -1,5 +1,7 @@ import inspect import json +import os +import ssl from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -7,7 +9,9 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest import redis import redis.asyncio as async_redis +from redis.asyncio.connection import AbstractConnection from redis.credentials import CredentialProvider +from redis.exceptions import RedisClusterException import litellm from litellm._redis import ( @@ -1960,3 +1964,965 @@ def test_async_sentinel_keeps_the_credential_provider_off_the_monitors(markers, master_kwargs = mock_sentinel_cls.return_value.master_for.call_args[1] assert isinstance(master_kwargs["credential_provider"], provider_cls) assert "password" not in master_kwargs + + +def _tls_view(connection: AbstractConnection) -> dict[str, object] | None: + if not isinstance(connection, async_redis.SSLConnection): + return None + context: Final = connection.ssl_context + return { + "ca_certs": context.ca_certs, + "certfile": context.certfile, + "keyfile": context.keyfile, + "verify_certificate": context.cert_reqs != ssl.CERT_NONE, + "check_hostname": context.check_hostname, + } + + +def _auth_view(kwargs: dict[str, object]) -> dict[str, object]: + provider: Final = kwargs.get("credential_provider") + match provider: + case None: + return {"kind": "static", "username": kwargs.get("username"), "password": kwargs.get("password")} + case GCPIAMCredentialProvider(): + return {"kind": "gcp_iam", "service_account": provider._gcp_service_account} + case AzureADCredentialProvider(): + return {"kind": "azure_ad", "username": provider._username} + case ElastiCacheIAMCredentialProvider(): + return { + "kind": "aws_iam", + "user_name": provider._user_name, + "cache_name": provider._cache_name, + "region": provider._region, + "serverless": provider._is_serverless, + } + case _: + return {"kind": "host"} + + +def _connection_view(connection: AbstractConnection, kwargs: dict[str, object]) -> dict[str, object]: + return { + "tls": _tls_view(connection), + "auth": _auth_view(kwargs), + "socket_timeout": connection.socket_timeout, + "socket_connect_timeout": connection.socket_connect_timeout, + } + + +def _resolution_view(kwargs: dict[str, object]) -> dict[str, object]: + url: Final = _credential_provider_auth_kwargs(_get_redis_client_logic(**kwargs)).get("url") + client: Final = get_redis_async_client(**kwargs) + if isinstance(client, async_redis.RedisCluster): + nodes: Final = list(client.nodes_manager.startup_nodes.values()) + return { + "mode": "cluster", + "nodes": [f"{node.host}:{node.port}" for node in nodes], + **_connection_view(nodes[0].acquire_connection(), client.connection_kwargs), + } + pool: Final = client.connection_pool + connection: Final = pool.make_connection() + if isinstance(pool, async_redis.SentinelConnectionPool): + sentinels: Final = [sentinel.connection_pool.connection_kwargs for sentinel in pool.sentinel_manager.sentinels] + return { + "mode": "sentinel", + "nodes": [f"{sentinel['host']}:{sentinel['port']}" for sentinel in sentinels], + "service_name": pool.service_name, + "sentinel_password": sentinels[0].get("password"), + "db": connection.db, + **_connection_view(connection, pool.connection_kwargs), + } + match connection: + case async_redis.UnixDomainSocketConnection(): + target = {"mode": "unix", "path": connection.path} + case _ if url is not None: + target = {"mode": "url", "url": url} + case _: + target = {"mode": "host", "host": connection.host, "port": connection.port, "db": connection.db} + return {**target, **_connection_view(connection, pool.connection_kwargs)} + + +def _expected_view(fields: dict[str, object]) -> dict[str, object]: + return { + "auth": {"kind": "static", "username": None, "password": None}, + "tls": None, + "socket_timeout": 5.0, + "socket_connect_timeout": 5.0, + **fields, + } + + +@pytest.mark.skipif( + redis.VERSION != (5, 3, 1), + reason="the Rust resolver in litellm-rust/crates/config pins redis-py 5.3.1, per its AGENTS.md", +) +@pytest.mark.parametrize( + ("kwargs", "environment", "expected"), + [ + pytest.param( + {"host": "redis"}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="host_defaults", + ), + pytest.param( + {"host": "redis", "port": "6380\n", "db": " 2 ", "socket_timeout": "1_0.5\n"}, + {}, + _expected_view( + { + "db": 2, + "host": "redis", + "mode": "host", + "port": 6380, + "socket_timeout": 10.5, + "socket_connect_timeout": 10.5, + } + ), + id="padded_and_underscored_numbers_parse_like_python_int_and_float", + ), + pytest.param( + {}, + {"REDIS_HOST": "redis", "REDIS_PORT": "6_380", "REDIS_DB": "3\n"}, + _expected_view({"db": 3, "host": "redis", "mode": "host", "port": 6380}), + id="padded_environment_numbers_parse_like_python_int", + ), + pytest.param( + {"startup_nodes": []}, + {"REDIS_CLUSTER_NODES": '[{"host": "node-a", "port": 7000}]'}, + _expected_view({"mode": "cluster", "nodes": ["node-a:7000"]}), + id="empty_cluster_nodes_use_the_environment", + ), + pytest.param( + {"sentinel_nodes": [], "service_name": "primary"}, + {"REDIS_SENTINEL_NODES": '[["sentinel-a", 26379]]'}, + _expected_view( + { + "db": 0, + "mode": "sentinel", + "nodes": ["sentinel-a:26379"], + "sentinel_password": None, + "service_name": "primary", + } + ), + id="empty_sentinel_nodes_use_the_environment", + ), + pytest.param( + {"sentinel_nodes": [["sentinel-a", 26379]], "service_name": ""}, + {"REDIS_SERVICE_NAME": "primary"}, + _expected_view( + { + "db": 0, + "mode": "sentinel", + "nodes": ["sentinel-a:26379"], + "sentinel_password": None, + "service_name": "primary", + } + ), + id="empty_service_name_uses_the_environment", + ), + pytest.param( + {"host": "redis", "ssl_cert_reqs": "strict"}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="plaintext_ignores_tls_requirements", + ), + pytest.param( + {"host": "redis", "ssl": "on"}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="an_ambiguous_flag_is_refused", + ), + pytest.param( + {"sentinel_nodes": [], "service_name": "primary"}, + {}, + { + "rejected": { + "message": "Both 'sentinel_nodes' and 'service_name' are required for Redis Sentinel.", + "type": "ValueError", + } + }, + id="empty_sentinel_nodes_without_the_environment_name_no_target", + ), + pytest.param( + {"host": "redis", "ssl": True, "ssl_ca_path": "/ca"}, + {}, + { + "rejected": { + "message": "Redis.__init__() got an unexpected keyword argument 'ssl_ca_path'", + "type": "TypeError", + } + }, + id="ssl_ca_path_is_unsupported", + ), + pytest.param( + { + "db": 2, + "host": "redis", + "password": "pass", + "port": 6380, + "socket_timeout": 1.5, + "ssl": True, + "username": "user", + }, + {}, + _expected_view( + { + "auth": {"kind": "static", "password": "pass", "username": "user"}, + "db": 2, + "host": "redis", + "mode": "host", + "port": 6380, + "socket_connect_timeout": 1.5, + "socket_timeout": 1.5, + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="host_with_every_basic_setting", + ), + pytest.param( + {}, + { + "REDIS_HOST": "environment", + "REDIS_PASSWORD": "environment-secret", + "REDIS_PORT": "6390", + "REDIS_SSL": "True", + }, + _expected_view( + { + "auth": {"kind": "static", "password": "environment-secret", "username": None}, + "db": 0, + "host": "environment", + "mode": "host", + "port": 6390, + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="environment_host_port_and_password", + ), + pytest.param( + {"db": "3", "host": "redis", "port": "6380", "socket_timeout": "2.5"}, + {}, + _expected_view( + { + "db": 3, + "host": "redis", + "mode": "host", + "port": 6380, + "socket_connect_timeout": 2.5, + "socket_timeout": 2.5, + } + ), + id="string_numbers_are_coerced", + ), + pytest.param( + {"host": "redis"}, + {"REDIS_SOCKET_TIMEOUT": "0.75"}, + _expected_view( + { + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + "socket_connect_timeout": 0.75, + "socket_timeout": 0.75, + } + ), + id="socket_timeout_from_the_environment", + ), + pytest.param( + {}, + {"REDIS_URL": "redis://environment:6379/1"}, + _expected_view({"mode": "url", "url": "redis://environment:6379/1"}), + id="environment_url", + ), + pytest.param( + {}, + {"REDIS_HOST": "ignored", "REDIS_URL": "redis://from-url:6379"}, + _expected_view({"mode": "url", "url": "redis://from-url:6379"}), + id="environment_url_beats_the_environment_host", + ), + pytest.param( + {"host": "configured"}, + {"REDIS_URL": "redis://environment:6379"}, + _expected_view({"db": 0, "host": "configured", "mode": "host", "port": 6379}), + id="an_explicit_host_drops_the_environment_url", + ), + pytest.param( + {"db": 4, "host": "dropped", "password": "dropped", "url": "rediss://cache:6380", "username": "user"}, + {}, + _expected_view( + { + "auth": {"kind": "static", "password": None, "username": "user"}, + "mode": "url", + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + "url": "rediss://cache:6380", + } + ), + id="a_url_drops_host_db_and_password_and_keeps_the_username", + ), + pytest.param( + { + "ssl_ca_certs": "/ca.pem", + "ssl_cert_reqs": "none", + "ssl_check_hostname": True, + "url": "rediss://cache:6380", + }, + {}, + _expected_view( + { + "mode": "url", + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + "url": "rediss://cache:6380", + } + ), + id="a_url_ignores_tls_settings_outside_the_url", + ), + pytest.param( + { + "password": "cluster-secret", + "startup_nodes": [{"host": "node-a", "port": 7000}], + "url": "redis://single:6379", + }, + {}, + _expected_view( + { + "auth": {"kind": "static", "password": "cluster-secret", "username": None}, + "mode": "cluster", + "nodes": ["node-a:7000"], + } + ), + id="startup_nodes_beat_a_url", + ), + pytest.param( + {}, + {"REDIS_CLUSTER_NODES": '[{"host": "node-a", "port": "7000"}, {"host": "node-b", "port": 7001}]'}, + _expected_view({"mode": "cluster", "nodes": ["node-a:7000", "node-b:7001"]}), + id="environment_cluster_nodes", + ), + pytest.param( + { + "ssl": "true", + "ssl_ca_certs": "/ca.pem", + "ssl_check_hostname": True, + "startup_nodes": [{"host": "node-a", "port": 7000}], + }, + {}, + _expected_view( + { + "mode": "cluster", + "nodes": ["node-a:7000"], + "tls": { + "ca_certs": "/ca.pem", + "certfile": None, + "check_hostname": True, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="cluster_tls_options", + ), + pytest.param( + { + "password": "data-secret", + "sentinel_nodes": [["sentinel-a", 26379], ["sentinel-b", 26380]], + "sentinel_password": "sentinel-secret", + "service_name": "primary", + }, + {}, + _expected_view( + { + "auth": {"kind": "static", "password": "data-secret", "username": None}, + "db": 0, + "mode": "sentinel", + "nodes": ["sentinel-a:26379", "sentinel-b:26380"], + "sentinel_password": "sentinel-secret", + "service_name": "primary", + } + ), + id="sentinel_from_settings", + ), + pytest.param( + {}, + { + "REDIS_SENTINEL_NODES": '[["sentinel-a", 26379]]', + "REDIS_SENTINEL_PASSWORD": "sentinel-secret", + "REDIS_SERVICE_NAME": "primary", + }, + _expected_view( + { + "db": 0, + "mode": "sentinel", + "nodes": ["sentinel-a:26379"], + "sentinel_password": "sentinel-secret", + "service_name": "primary", + } + ), + id="sentinel_from_the_environment", + ), + pytest.param( + {"host": "ignored", "sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary"}, + {}, + { + "rejected": { + "message": "redis.asyncio.client.Redis() got multiple values for keyword argument 'host'", + "type": "TypeError", + } + }, + id="a_host_breaks_sentinel", + ), + pytest.param( + {"sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary", "url": "redis://single:6379"}, + {}, + _expected_view({"mode": "url", "url": "redis://single:6379"}), + id="a_url_beats_sentinels", + ), + pytest.param( + { + "aws_iam_auth": True, + "credential_provider": True, + "gcp_service_account": "account", + "host": "redis", + "password": "dropped", + }, + {"REDIS_AZURE_AD_TOKEN": "true"}, + _expected_view({"auth": {"kind": "host"}, "db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="an_explicit_credential_provider_beats_cloud_auth", + ), + pytest.param( + {"aws_iam_auth": True, "azure_redis_ad_token": "True", "host": "redis", "password": "dropped"}, + {"REDIS_GCP_SERVICE_ACCOUNT": "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com"}, + _expected_view( + { + "auth": { + "kind": "gcp_iam", + "service_account": "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com", + }, + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + } + ), + id="gcp_beats_azure_and_aws", + ), + pytest.param( + { + "gcp_service_account": "account", + "gcp_ssl_ca_certs": "/gcp-ca.pem", + "host": "redis", + "ssl": True, + "ssl_ca_certs": "/configured-ca.pem", + }, + {}, + _expected_view( + { + "auth": {"kind": "gcp_iam", "service_account": "account"}, + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + "tls": { + "ca_certs": "/gcp-ca.pem", + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="gcp_ca_certs_replace_the_configured_ones_under_ssl", + ), + pytest.param( + { + "aws_iam_auth": True, + "azure_client_id": "client", + "azure_client_secret": "secret", + "azure_redis_ad_token": True, + "azure_tenant_id": "tenant", + "host": "redis", + "username": "dropped", + }, + {"REDIS_USERNAME": "environment-user"}, + _expected_view( + { + "auth": {"kind": "azure_ad", "username": "environment-user"}, + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + } + ), + id="azure_beats_aws_and_takes_the_environment_username", + ), + pytest.param( + { + "azure_client_id": "client", + "azure_client_secret": "secret", + "azure_redis_ad_token": True, + "azure_tenant_id": "tenant", + "host": "redis", + }, + {}, + _expected_view( + {"auth": {"kind": "azure_ad", "username": None}, "db": 0, "host": "redis", "mode": "host", "port": 6379} + ), + id="azure_without_a_username_sends_only_the_token", + ), + pytest.param( + { + "azure_client_id": "client", + "azure_client_secret": "secret", + "azure_redis_ad_token": "1", + "azure_tenant_id": "tenant", + "host": "redis", + }, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="azure_needs_the_literal_true", + ), + pytest.param( + { + "aws_iam_auth": "yes", + "aws_iam_cache_name": "cache-1", + "aws_iam_user_name": "iam-user", + "host": "cache-1", + "password": "dropped", + "ssl": True, + }, + {"AWS_DEFAULT_REGION": "us-east-1"}, + _expected_view( + { + "auth": { + "cache_name": "cache-1", + "kind": "aws_iam", + "region": "us-east-1", + "serverless": False, + "user_name": "iam-user", + }, + "db": 0, + "host": "cache-1", + "mode": "host", + "port": 6379, + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="aws_iam_over_tls_with_the_default_region", + ), + pytest.param( + { + "aws_iam_auth": True, + "aws_iam_cache_name": "cache-1", + "aws_iam_user_name": "iam-user", + "host": "cache-1", + "ssl": True, + }, + {"AWS_DEFAULT_REGION": "us-east-1", "AWS_REGION": "eu-central-1"}, + _expected_view( + { + "auth": { + "cache_name": "cache-1", + "kind": "aws_iam", + "region": "eu-central-1", + "serverless": False, + "user_name": "iam-user", + }, + "db": 0, + "host": "cache-1", + "mode": "host", + "port": 6379, + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="aws_region_beats_the_default_region", + ), + pytest.param( + { + "aws_iam_auth": True, + "aws_iam_cache_name": "cache-1", + "aws_iam_region": "eu-west-1", + "aws_iam_serverless": True, + "aws_iam_user_name": "iam-user", + "url": "rediss://user:pass@cache-1:6379", + }, + {}, + _expected_view( + { + "auth": { + "cache_name": "cache-1", + "kind": "aws_iam", + "region": "eu-west-1", + "serverless": True, + "user_name": "iam-user", + }, + "mode": "url", + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + "url": "rediss://cache-1:6379", + } + ), + id="aws_iam_serverless_over_a_rediss_url_drops_its_userinfo", + ), + pytest.param( + { + "aws_iam_auth": True, + "aws_iam_cache_name": "cache-1", + "aws_iam_region": "us-east-1", + "aws_iam_user_name": "iam-user", + "host": "cache-1", + }, + {}, + {"rejected": {"message": "AWS ElastiCache IAM Redis authentication requires TLS", "type": "ValueError"}}, + id="aws_iam_without_tls_fails", + ), + pytest.param( + { + "aws_iam_auth": True, + "aws_iam_cache_name": "cache-1", + "aws_iam_user_name": "iam-user", + "host": "cache-1", + "ssl": True, + }, + {}, + { + "rejected": { + "message": "AWS ElastiCache IAM Redis authentication requires: aws_iam_region", + "type": "ValueError", + } + }, + id="aws_iam_without_a_region_fails", + ), + pytest.param( + { + "host": "redis", + "ssl": True, + "ssl_ca_certs": "/ca.pem", + "ssl_cert_reqs": "none", + "ssl_check_hostname": True, + }, + {}, + _expected_view( + { + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + "tls": { + "ca_certs": "/ca.pem", + "certfile": None, + "check_hostname": True, + "keyfile": None, + "verify_certificate": False, + }, + } + ), + id="tls_options_from_settings", + ), + pytest.param( + {"host": "redis"}, + {"REDIS_SSL": "true", "REDIS_SSL_CA_CERTS": "/env-ca.pem", "REDIS_SSL_CERT_REQS": "optional"}, + _expected_view( + { + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + "tls": { + "ca_certs": "/env-ca.pem", + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="tls_options_from_the_environment", + ), + pytest.param( + {"host": "redis", "socket_connect_timeout": 0.25, "socket_timeout": 2.5}, + {}, + _expected_view( + { + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + "socket_connect_timeout": 0.25, + "socket_timeout": 2.5, + } + ), + id="connect_timeout_is_separate_from_the_socket_timeout", + ), + pytest.param( + {"host": "os.environ/COORD_HOST", "password": "os.environ/COORD_PASSWORD"}, + {"COORD_HOST": "referenced", "COORD_PASSWORD": "referenced-secret"}, + _expected_view( + { + "auth": {"kind": "static", "password": "referenced-secret", "username": None}, + "db": 0, + "host": "referenced", + "mode": "host", + "port": 6379, + } + ), + id="environment_references_resolve", + ), + pytest.param( + {"port": 6379}, + {}, + {"rejected": {"message": "Either 'host' or 'url' must be specified for redis.", "type": "ValueError"}}, + id="no_target_fails", + ), + pytest.param( + {"host": "redis", "ssl": True, "ssl_certfile": "/client.pem", "ssl_keyfile": "/client.key"}, + {}, + _expected_view( + { + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + "tls": { + "ca_certs": None, + "certfile": "/client.pem", + "check_hostname": False, + "keyfile": "/client.key", + "verify_certificate": True, + }, + } + ), + id="client_certificates", + ), + pytest.param( + {"url": "rediss://cache:6380/0?ssl_cert_reqs=none&ssl_ca_certs=/ca.pem&ssl_check_hostname=yes"}, + {}, + _expected_view( + { + "mode": "url", + "tls": { + "ca_certs": "/ca.pem", + "certfile": None, + "check_hostname": True, + "keyfile": None, + "verify_certificate": False, + }, + "url": "rediss://cache:6380/0?ssl_cert_reqs=none&ssl_ca_certs=/ca.pem&ssl_check_hostname=yes", + } + ), + id="a_rediss_url_takes_tls_options_from_its_query", + ), + pytest.param( + {"url": "redis://cache:6379/0?socket_timeout=2"}, + {}, + _expected_view( + { + "mode": "url", + "socket_connect_timeout": 2.0, + "socket_timeout": 2.0, + "url": "redis://cache:6379/0?socket_timeout=2", + } + ), + id="a_url_query_setting_without_a_redis_rs_equivalent", + ), + pytest.param( + {"db": "99999999999999999999", "ssl_cert_reqs": "strict", "url": "rediss://cache:6380"}, + {}, + _expected_view( + { + "mode": "url", + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + "url": "rediss://cache:6380", + } + ), + id="tls_settings_outside_a_url_never_fail_it", + ), + pytest.param( + {"db": 99999999999, "startup_nodes": [{"host": "node-a", "port": 7000}]}, + {}, + { + "rejected": { + "message": "Argument 'db' must be 0 or None in cluster mode", + "type": "RedisClusterException", + } + }, + id="db_is_not_read_for_a_cluster", + ), + pytest.param( + {"db": True, "startup_nodes": [{"host": "node-a", "port": 7000}]}, + {}, + { + "rejected": { + "message": "Argument 'db' must be 0 or None in cluster mode", + "type": "RedisClusterException", + } + }, + id="cluster_db_true", + ), + pytest.param( + {"db": False, "startup_nodes": [{"host": "node-a", "port": 7000}]}, + {}, + _expected_view({"mode": "cluster", "nodes": ["node-a:7000"]}), + id="cluster_db_false", + ), + pytest.param( + {"db": 0.5, "startup_nodes": [{"host": "node-a", "port": 7000}]}, + {}, + { + "rejected": { + "message": "Argument 'db' must be 0 or None in cluster mode", + "type": "RedisClusterException", + } + }, + id="cluster_db_fraction", + ), + pytest.param( + {"db": 0.0, "startup_nodes": [{"host": "node-a", "port": 7000}]}, + {}, + _expected_view({"mode": "cluster", "nodes": ["node-a:7000"]}), + id="cluster_db_float_zero", + ), + pytest.param( + {"db": [1], "startup_nodes": [{"host": "node-a", "port": 7000}]}, + {}, + { + "rejected": { + "message": "Argument 'db' must be 0 or None in cluster mode", + "type": "RedisClusterException", + } + }, + id="cluster_db_nonempty_list", + ), + pytest.param( + {"db": [], "startup_nodes": [{"host": "node-a", "port": 7000}]}, + {}, + _expected_view({"mode": "cluster", "nodes": ["node-a:7000"]}), + id="cluster_db_empty_list", + ), + pytest.param( + {"host": "redis", "ssl": True, "ssl_ciphers": "HIGH"}, + {}, + _expected_view( + { + "db": 0, + "host": "redis", + "mode": "host", + "port": 6379, + "tls": { + "ca_certs": None, + "certfile": None, + "check_hostname": False, + "keyfile": None, + "verify_certificate": True, + }, + } + ), + id="ssl_ciphers", + ), + pytest.param( + {"host": "redis", "retry_on_timeout": True}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="retry_on_timeout", + ), + pytest.param( + {"host": "redis", "port": "abc"}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="malformed_port", + ), + pytest.param( + {"db": "first", "host": "redis"}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="malformed_db", + ), + pytest.param( + {"host": "redis", "socket_timeout": "slow"}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="socket_timeout_not_a_number", + ), + pytest.param( + {}, + {"REDIS_HOST": "redis", "REDIS_PORT": "abc"}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="malformed_port_from_the_environment", + ), + pytest.param( + {"host": "redis", "protocol": 3}, + {}, + _expected_view({"db": 0, "host": "redis", "mode": "host", "port": 6379}), + id="protocol_3", + ), + pytest.param( + {"host": "redis"}, + {"REDIS_RETRY_ON_ERROR": "true"}, + {"rejected": {"message": "'bool' object is not iterable", "type": "TypeError"}}, + id="retry_on_error_from_the_environment", + ), + pytest.param( + {"host": "redis", "unix_socket_path": "/tmp/redis.sock"}, + {}, + _expected_view({"mode": "unix", "path": "/tmp/redis.sock"}), + id="unix_socket_path", + ), + ], +) +def test_redis_settings_resolve_like_the_rust_resolver_expects(monkeypatch, kwargs, environment, expected): + for name in [name for name in os.environ if name.startswith(("REDIS_", "AZURE_", "AWS_"))]: + monkeypatch.delenv(name) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + try: + outcome = _resolution_view(kwargs) + except (ValueError, TypeError, RedisClusterException) as error: + outcome = {"rejected": {"type": type(error).__name__, "message": str(error)}} + + assert outcome == expected