diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4eebd2145ae..8573049d0a1 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -982,6 +993,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bollard" version = "0.20.2" @@ -1134,6 +1154,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.3.0" @@ -1210,6 +1239,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.7" @@ -1774,6 +1813,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher", +] + [[package]] name = "digest" version = "0.10.7" @@ -3105,6 +3153,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -4372,6 +4430,7 @@ dependencies = [ "litellm-redis-types", "litellm-secret-value", "pem", + "pkcs8", "redis", "rstest", "rustls 0.23.42", @@ -4403,6 +4462,7 @@ dependencies = [ name = "litellm-redis-testing" version = "0.1.0" dependencies = [ + "pkcs8", "rcgen", "tempfile", "testcontainers-modules", @@ -5283,6 +5343,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pem" version = "4.0.0" @@ -5393,6 +5463,22 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "des", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -5400,6 +5486,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core 0.6.4", "spki", ] @@ -6408,6 +6496,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6471,6 +6568,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sct" version = "0.7.1" diff --git a/litellm-rust/crates/config/AGENTS.md b/litellm-rust/crates/config/AGENTS.md index 719083ff572..2c43a88a5c9 100644 --- a/litellm-rust/crates/config/AGENTS.md +++ b/litellm-rust/crates/config/AGENTS.md @@ -7,7 +7,7 @@ - `resolve` takes a `SecretSnapshot` and an environment `Lookup`. The snapshot holds every name `ConfiguredRedis::secret_names` lists, read the way Python's `get_secret` reads it (secret manager, then environment). The host fills it once before resolving, python-bridge through `get_secret` and rust-proxy from its environment, so the resolver never calls back into the host. The environment lookup is only for what Python reads straight from `os.environ` (`REDIS_USERNAME` for Azure). Reading a name the snapshot does not list fails a debug assertion - Every Redis setting Python reads is one row in `SETTINGS` in `src/redis/keys.rs`: how Rust handles it (read, or warn and drop), whether `_get_redis_env_kwarg_mapping` reads it as `REDIS_`, and the names `_get_redis_client_logic` falls back to. The env layer, the ignored list and `secret_names` all come from that table. A read row is also a field of `Options` - Lookups go through `litellm_core_utils::settings::Lookup`, never a closure type of our own -- A url carries its own TLS. redis-py's url allowlist is the plain `Connection` signature, so every `ssl_*` setting outside the url is dropped, and a credential provider strips the url's userinfo. A `rediss://` url's `ssl_cert_reqs`, `ssl_ca_certs`, `ssl_ca_data`, `ssl_ca_path`, `ssl_certfile`, `ssl_keyfile` and `ssl_check_hostname` query parameters are read like `parse_url` reads them +- A url carries its own TLS. redis-py's url allowlist is the plain `Connection` signature, so every `ssl_*` setting outside the url is dropped, and a credential provider strips the url's userinfo. A `rediss://` url's `ssl_cert_reqs`, `ssl_ca_certs`, `ssl_ca_data`, `ssl_ca_path`, `ssl_certfile`, `ssl_keyfile`, `ssl_password` and `ssl_check_hostname` query parameters are read like `parse_url` reads them - TLS options and `db` are only read for the endpoint that uses them, so a bad `ssl_cert_reqs` never fails a url target. A cluster with a non-zero `db`, and Sentinel with a `host` or `port`, are errors because redis-py raises building those clients - An empty value (`""`, `[]`, `{}`) is the same as an unset one, so it falls through to its environment fallback the way Python's `x or get_secret(...)` does. Nulls depend on the surface. `coordination_redis` drops every null like `model_dump(exclude_none=True)`, `cache_params` keeps a null as an explicit `None` that hides the `REDIS_*` value unless `Cache` or `RedisCache` names that parameter, and `RedisCache.redis_kwargs` keeps every null. A missing `os.environ/` reference is a null, as `load_config` resolves it - Precedence matches Python. The target is cluster, then URL, then Sentinel (`sentinel_nodes` plus `service_name`), then host. Auth is an explicit `credential_provider`, then GCP, then Azure, then AWS, and the first one wins with a warning @@ -15,8 +15,8 @@ - Flags (`ssl`, `ssl_check_hostname`, `aws_iam_auth`, `aws_iam_serverless`) are read one way on every surface. `true`, `1` and `yes` are true and `false`, `0`, `no`, `off`, `f` and `n` are false, the text redis-py's `_str_to_bool` and the proxy's pydantic models agree on. Any other text is `RedisError::Invalid`, so a TLS setting is never guessed - `RedisSettings::coordination` mirrors the proxy's `CoordinationRedisParams` pydantic model, which rejects a malformed `port` before redis-py coerces anything - The resolver never fails on a setting it cannot honor. A `Warn` row is dropped and returned as an `Ignored` naming its source (`ssl_ciphers` or `REDIS_SSL_CIPHERS`) and the consequence from `keys::consequence`, because a Python deployment that worked must keep working when its stores move to Rust. Only a value that is malformed (`RedisError::Invalid`) or a target that is missing fails. Promoting a `Warn` row to `Read` is then a pure improvement with no behavior cliff - - `ssl_ciphers`, `ssl_min_version` above TLS 1.2, `ssl_password`, the OCSP options, `connection_pool`, `event_dispatcher`, `redis_connect_func` and `retry` are ignored. Ignoring OCSP connects without revocation checks, and the consequence says so - - `ssl_ca_data` and `ssl_ca_path` are read into `TlsOptions` and trusted next to `ssl_ca_certs`. `unix_socket_path` resolves to `Target::Unix`, which like redis-py's `UnixDomainSocketConnection` replaces host, port and TLS. Python still requires `host` next to it, so Rust does too + - `ssl_ciphers`, `ssl_min_version` above TLS 1.2, the OCSP options, `connection_pool`, `event_dispatcher`, `redis_connect_func` and `retry` are ignored. Ignoring OCSP connects without revocation checks, and the consequence says so + - `ssl_ca_data` and `ssl_ca_path` are read into `TlsOptions` and trusted next to `ssl_ca_certs`. `ssl_password` is read into `TlsOptions::key_password`, which `redis-client` uses to decrypt a PKCS#8 key like the sync client's `load_cert_chain(password=...)`. redis-py 5.3.1's async client refuses it, so this is a superset of the path the proxy uses, like `ssl_ca_path` `unix_socket_path` resolves to `Target::Unix`, which like redis-py's `UnixDomainSocketConnection` replaces host, port and TLS. Python still requires `host` next to it, so Rust does too - A url's `?db=`, `?username=` and `?password=` are read like `parse_url` reads them, so `?db=` wins over the path and the userinfo wins over `?username=` and `?password=`. Any other query parameter except the TLS ones on a `rediss://` url is ignored naming `url ?`. redis-py turns each one into a client kwarg - `retry_on_timeout`, `retry_on_error`, `protocol`, `cache`, `cache_config`, `socket_keepalive` and `max_connections` are ignored. Cluster clients still turn TCP keepalive on, like Python's async cluster default - `health_check_interval` is read as an `int` of seconds like `_coerce_redis_kwargs_types` into `HealthCheck`. Zero is `Disabled` and stays off on every target, a negative interval checks before every command like redis-py, and only `Unset` takes a default. `redis-connect` gives an unset cluster Python's 25s default and `redis-client` pings every node before the next command once the client sat idle that long, outside the breaker and the call events diff --git a/litellm-rust/crates/config/src/redis/keys.rs b/litellm-rust/crates/config/src/redis/keys.rs index d5fc698ff4b..7d0bac71cc5 100644 --- a/litellm-rust/crates/config/src/redis/keys.rs +++ b/litellm-rust/crates/config/src/redis/keys.rs @@ -149,7 +149,7 @@ pub(super) const SETTINGS: &[Setting] = &[ Setting::new("ssl_min_version", Handling::Warn, true, &[]), Setting::new("ssl_ocsp_context", Handling::Warn, true, &[]), Setting::new("ssl_ocsp_expected_cert", Handling::Warn, true, &[]), - Setting::new("ssl_password", Handling::Warn, true, &[]), + Setting::new("ssl_password", Handling::Read, true, &[]), Setting::new("ssl_validate_ocsp", Handling::Warn, true, &[]), Setting::new("ssl_validate_ocsp_stapled", Handling::Warn, true, &[]), Setting::new( @@ -224,7 +224,6 @@ fn consequence(kwarg: &str) -> &'static str { "ssl_ciphers" | "ssl_min_version" => { "rustls negotiates TLS 1.2+ with its own ECDHE AEAD suites" } - "ssl_password" => "an encrypted client key cannot be read, so the TLS handshake fails", "credential_provider" | "redis_connect_func" => { "the Python object is not callable from Rust, so static username and password are used" } diff --git a/litellm-rust/crates/config/src/redis/options.rs b/litellm-rust/crates/config/src/redis/options.rs index 24b6b6ba496..70af41ad074 100644 --- a/litellm-rust/crates/config/src/redis/options.rs +++ b/litellm-rust/crates/config/src/redis/options.rs @@ -75,6 +75,7 @@ pub(super) struct TlsInput { pub(super) ca_path: Option, pub(super) certfile: Option, pub(super) keyfile: Option, + pub(super) key_password: Option, pub(super) check_hostname: Option, } @@ -172,6 +173,7 @@ impl Options { ca_path: get("ssl_ca_path"), certfile: get("ssl_certfile"), keyfile: get("ssl_keyfile"), + key_password: get("ssl_password"), check_hostname: get("ssl_check_hostname"), }, namespace: get("namespace"), diff --git a/litellm-rust/crates/config/src/redis/tls.rs b/litellm-rust/crates/config/src/redis/tls.rs index 5444fe169fe..d9daaef6b67 100644 --- a/litellm-rust/crates/config/src/redis/tls.rs +++ b/litellm-rust/crates/config/src/redis/tls.rs @@ -20,6 +20,7 @@ const URL_TLS_SETTINGS: &[&str] = &[ "ssl_ca_path", "ssl_certfile", "ssl_keyfile", + "ssl_password", "ssl_check_hostname", ]; @@ -34,6 +35,7 @@ pub(super) fn tls_options( ca_path: text(input.ca_path.as_ref()).map(Into::into), certfile: text(input.certfile.as_ref()).map(Into::into), keyfile: text(input.keyfile.as_ref()).map(Into::into), + key_password: secret(input.key_password.as_ref()), verify_certificate: verify_certificate(input.cert_reqs.as_ref())?, check_hostname: flag(input.check_hostname.as_ref())?, }) @@ -99,6 +101,7 @@ pub(super) fn url_query(url: &str) -> Result { ca_path: text(get("ssl_ca_path").as_ref()).map(Into::into), certfile: text(get("ssl_certfile").as_ref()).map(Into::into), keyfile: text(get("ssl_keyfile").as_ref()).map(Into::into), + key_password: secret(get("ssl_password").as_ref()), verify_certificate: verify_certificate(get("ssl_cert_reqs").as_ref())?, check_hostname: text(get("ssl_check_hostname").as_ref()) .is_some_and(|text| url_bool(&text)), diff --git a/litellm-rust/crates/config/tests/redis.rs b/litellm-rust/crates/config/tests/redis.rs index 5479ec4d55d..c3dbce45ce1 100644 --- a/litellm-rust/crates/config/tests/redis.rs +++ b/litellm-rust/crates/config/tests/redis.rs @@ -7,6 +7,7 @@ use litellm_redis_types::{ Auth, AwsIamAuth, AzureAdAuth, HealthCheck, Node, ProviderHandle, Resolution, StaticAuth, Target, TlsOptions, }; +use litellm_secret_value::SecretValue; use rstest::rstest; use serde_json::json; @@ -327,22 +328,22 @@ fn incomplete_aws_iam_auth_fails(#[case] yaml: &str, #[case] message: &str) { #[case::defaults_verify_the_certificate_but_not_the_hostname( " ssl: true", &[], - Some(TlsOptions { ca_certs: None, ca_data: None, ca_path: None, certfile: None, keyfile: None, verify_certificate: true, check_hostname: false }) + Some(TlsOptions { ca_certs: None, ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: None, verify_certificate: true, check_hostname: false }) )] #[case::every_option( - " ssl: true\n ssl_ca_certs: /ca.pem\n ssl_certfile: /client.pem\n ssl_keyfile: /client.key\n ssl_cert_reqs: none\n ssl_check_hostname: true", + " ssl: true\n ssl_ca_certs: /ca.pem\n ssl_certfile: /client.pem\n ssl_keyfile: /client.key\n ssl_password: key-secret\n ssl_cert_reqs: none\n ssl_check_hostname: true", &[], - Some(TlsOptions { ca_certs: Some("/ca.pem".into()), ca_data: None, ca_path: None, certfile: Some("/client.pem".into()), keyfile: Some("/client.key".into()), verify_certificate: false, check_hostname: true }) + Some(TlsOptions { ca_certs: Some("/ca.pem".into()), ca_data: None, ca_path: None, certfile: Some("/client.pem".into()), keyfile: Some("/client.key".into()), key_password: Some(SecretValue::new("key-secret")), verify_certificate: false, check_hostname: true }) )] #[case::options_from_the_environment( " ssl: true", - &[("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional")], - Some(TlsOptions { ca_certs: Some("/env-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, verify_certificate: true, check_hostname: false }) + &[("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional"), ("REDIS_SSL_PASSWORD", "env-key-secret")], + Some(TlsOptions { ca_certs: Some("/env-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: Some(SecretValue::new("env-key-secret")), verify_certificate: true, check_hostname: false }) )] #[case::gcp_ca_certs_replace_the_configured_ones( " ssl: true\n ssl_ca_certs: /ca.pem\n gcp_service_account: account\n gcp_ssl_ca_certs: /gcp-ca.pem", &[], - Some(TlsOptions { ca_certs: Some("/gcp-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, verify_certificate: true, check_hostname: false }) + Some(TlsOptions { ca_certs: Some("/gcp-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: None, verify_certificate: true, check_hostname: false }) )] #[case::options_without_ssl_stay_plaintext(" ssl_ca_certs: /ca.pem", &[], None)] fn tls_options_resolve_like_redis_py( @@ -357,6 +358,22 @@ fn tls_options_resolve_like_redis_py( assert!(matches!(resolved.target, Target::Standalone(host) if host.tls == expected)); } +#[rstest] +fn a_rediss_url_reads_ssl_password_from_its_query() { + let resolved = resolve_kwargs( + json!({"url": "rediss://cache:6380/0?ssl_password=query-key-secret"}), + &HashMap::new(), + ) + .unwrap(); + + let Target::Url(parsed) = resolved.target else { + panic!("expected a url target, got {:?}", resolved.target); + }; + let key_password = parsed.tls.and_then(|tls| tls.key_password); + assert_eq!(expose(key_password.as_ref()), Some("query-key-secret")); + assert!(resolved.ignored.is_empty(), "{:?}", resolved.ignored); +} + #[rstest] fn an_unknown_cert_requirement_fails() { let error = resolve( @@ -958,6 +975,11 @@ fn expected_view(fields: serde_json::Value) -> serde_json::Value { &[], Ok(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": "/ca", "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), )] +#[case::ssl_password_is_unsupported( + json!({"host": "redis", "ssl": true, "ssl_certfile": "/client.pem", "ssl_password": "key-secret"}), + &[], + Ok(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": "/client.pem", "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] #[case::host_with_every_basic_setting( json!({"db": 2, "host": "redis", "password": "pass", "port": 6380, "socket_timeout": 1.5, "ssl": true, "username": "user"}), &[], diff --git a/litellm-rust/crates/redis-client/AGENTS.md b/litellm-rust/crates/redis-client/AGENTS.md index c53f71db73c..d2880ff5dd2 100644 --- a/litellm-rust/crates/redis-client/AGENTS.md +++ b/litellm-rust/crates/redis-client/AGENTS.md @@ -49,6 +49,7 @@ ## TLS, credentials and Sentinel - `TlsOptions` follows redis-py 5.3.1. It verifies the certificate by default, skips the hostname check by default (`ssl_check_hostname=False`), and trusts the system roots plus `ca_certs` +- rustls only reads unencrypted keys, so a PKCS#8 `ENCRYPTED PRIVATE KEY` is decrypted with `key_password` before it reaches redis-rs. A legacy `Proc-Type: 4,ENCRYPTED` PEM key is refused with `KeyError::LegacyEncryption` rather than decrypted by hand-rolled OpenSSL key derivation - redis-rs loads the system trust store for every TLS connection and fails if that fails, even with `ca_certs` set. Where the macOS keychain is blocked (agent sandboxes), run the TLS tests with `SSL_CERT_FILE=/etc/ssl/cert.pem` - `Credentials` are static and sent at connect. URL userinfo wins over them, like redis-py's `from_url` - Sentinel discovers the master on each new connection and drops it on a transport failure or READONLY, like redis-py's `SentinelConnectionPool` diff --git a/litellm-rust/crates/redis-client/Cargo.toml b/litellm-rust/crates/redis-client/Cargo.toml index d3ec1cdd728..f1245bb0b17 100644 --- a/litellm-rust/crates/redis-client/Cargo.toml +++ b/litellm-rust/crates/redis-client/Cargo.toml @@ -20,6 +20,7 @@ redis = { workspace = true, features = [ "tokio-rustls-comp", ] } pem = "4.0.0" +pkcs8 = { version = "0.10.2", features = ["3des", "encryption", "std"] } rustls-native-certs.workspace = true socket2 = "0.6.5" thiserror.workspace = true diff --git a/litellm-rust/crates/redis-client/src/config.rs b/litellm-rust/crates/redis-client/src/config.rs index 4dc114110b7..73b57fd76d0 100644 --- a/litellm-rust/crates/redis-client/src/config.rs +++ b/litellm-rust/crates/redis-client/src/config.rs @@ -8,7 +8,9 @@ use redis::{ use litellm_circuit_breaker::Breaker; use litellm_secret_value::SecretValue; -use crate::{ConnectError, Namespace, RedisEvents}; +use pkcs8::EncryptedPrivateKeyInfo; + +use crate::{ConnectError, KeyError, Namespace, RedisEvents}; use litellm_redis_types::{Cluster, RedisUrl, Target, TlsOptions}; // litellm/constants.py REDIS_SOCKET_TIMEOUT, which _coerce_redis_kwargs_types sets when socket_timeout is unset @@ -72,7 +74,7 @@ pub(crate) fn tls_certificates(tls: &TlsOptions) -> Result Some(ClientTlsConfig { client_cert: read(certfile)?, - client_key: read(keyfile.as_ref().unwrap_or(certfile))?, + client_key: client_key(keyfile.as_ref().unwrap_or(certfile), tls.key_password.as_ref())?, }), (None, _) => None, }; @@ -152,6 +154,43 @@ fn with_system_roots(configured: Vec) -> Vec { .collect() } +/// The key as unencrypted PEM, decrypting a PKCS#8 `ENCRYPTED PRIVATE KEY` with `ssl_password` the way +/// `load_cert_chain(password=...)` does, since rustls only reads unencrypted keys. Like OpenSSL, a password next to +/// an unencrypted key is unused +fn client_key(path: &PathBuf, password: Option<&SecretValue>) -> Result, ConnectError> { + let contents = read(path)?; + let key_error = |reason| ConnectError::TlsKey { + path: path.display().to_string(), + reason, + }; + let Ok(sections) = pem::parse_many(&contents) else { + return Ok(contents); + }; + let encrypted = sections + .iter() + .find(|section| section.tag() == "ENCRYPTED PRIVATE KEY"); + match (encrypted, password) { + (Some(_), None) => Err(key_error(KeyError::MissingPassword)), + (Some(section), Some(password)) => { + let decrypted = EncryptedPrivateKeyInfo::try_from(section.contents()) + .and_then(|info| info.decrypt(password.expose())) + .map_err(|source| key_error(KeyError::Decrypt(source)))?; + Ok(pem::encode(&pem::Pem::new("PRIVATE KEY", decrypted.as_bytes())).into_bytes()) + } + (None, _) if sections.iter().any(is_legacy_encrypted) => { + Err(key_error(KeyError::LegacyEncryption)) + } + (None, _) => Ok(contents), + } +} + +fn is_legacy_encrypted(section: &pem::Pem) -> bool { + section + .headers() + .get("Proc-Type") + .is_some_and(|proc_type| proc_type.contains("ENCRYPTED")) +} + fn read(path: &PathBuf) -> Result, ConnectError> { std::fs::read(path).map_err(|source| ConnectError::TlsFile { path: path.display().to_string(), diff --git a/litellm-rust/crates/redis-client/src/connection.rs b/litellm-rust/crates/redis-client/src/connection.rs index 334c38f0f3f..3043fe679d0 100644 --- a/litellm-rust/crates/redis-client/src/connection.rs +++ b/litellm-rust/crates/redis-client/src/connection.rs @@ -293,6 +293,11 @@ impl SentinelConnection { "reading a Redis TLS file", path, )), + ConnectError::TlsKey { path, reason } => RedisError::from(( + ErrorKind::InvalidClientConfig, + "reading the Redis TLS key", + format!("{path}: {reason}"), + )), }, )?; redis::Client::open(info) diff --git a/litellm-rust/crates/redis-client/src/error.rs b/litellm-rust/crates/redis-client/src/error.rs index 81169498926..54e55615e60 100644 --- a/litellm-rust/crates/redis-client/src/error.rs +++ b/litellm-rust/crates/redis-client/src/error.rs @@ -22,6 +22,24 @@ pub enum ConnectError { #[source] source: std::io::Error, }, + #[error("reading the Redis TLS key {path}")] + TlsKey { + path: String, + #[source] + reason: KeyError, + }, +} + +#[derive(Debug, thiserror::Error)] +pub enum KeyError { + #[error("the key is encrypted, so ssl_password is required")] + MissingPassword, + #[error("ssl_password does not decrypt the key, or the key uses an unsupported cipher")] + Decrypt(#[source] pkcs8::Error), + #[error( + "the key uses legacy OpenSSL PEM encryption, which Rust cannot read. Convert it to PKCS#8 with `openssl pkcs8 -topk8`" + )] + LegacyEncryption, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/litellm-rust/crates/redis-client/src/lib.rs b/litellm-rust/crates/redis-client/src/lib.rs index 1459c0e8db9..14d859c7d67 100644 --- a/litellm-rust/crates/redis-client/src/lib.rs +++ b/litellm-rust/crates/redis-client/src/lib.rs @@ -11,7 +11,7 @@ mod script; pub use client::{RedisClient, Ttl}; pub use commands::{Amount, Expiry}; pub use config::{Credentials, DEFAULT_TIMEOUT, RedisConfig, Timeouts}; -pub use error::{ConnectError, Error}; +pub use error::{ConnectError, Error, KeyError}; pub use events::{CallEvent, FailureKind, Operation, Outcome, RedisEvents}; pub use litellm_redis_types::{ Cluster, Node, RedisUrl, Sentinel, Standalone, StaticAuth, Target, TlsOptions, UnixSocket, diff --git a/litellm-rust/crates/redis-client/tests/config.rs b/litellm-rust/crates/redis-client/tests/config.rs index e23e153128c..438c1b0b26e 100644 --- a/litellm-rust/crates/redis-client/tests/config.rs +++ b/litellm-rust/crates/redis-client/tests/config.rs @@ -33,31 +33,28 @@ fn directory_holding(ca: &std::path::Path) -> std::path::PathBuf { directory } -async fn ping_over_tls( - require_client_certificate: bool, - host: &str, - ca: Ca, - verify_certificate: bool, - check_hostname: bool, - client_certificate: bool, -) -> Result<(), litellm_redis_client::Error> { +#[derive(Clone, Copy, Debug)] +enum ClientKey { + Absent, + Plain, + Encrypted { password: Option<&'static str> }, +} + +async fn tls_server(require_client_certificate: bool) -> litellm_redis_testing::RedisTlsServer { if rustls::crypto::CryptoProvider::get_default().is_none() { let _ = rustls::crypto::ring::default_provider().install_default(); } - let server = litellm_redis_testing::RedisTlsServer::start(require_client_certificate) + litellm_redis_testing::RedisTlsServer::start(require_client_certificate) .await - .unwrap(); - let files = server.files(); - let tls = TlsOptions { - ca_certs: matches!(ca, Ca::Trusted).then(|| files.ca.clone()), - ca_data: matches!(ca, Ca::TrustedData).then(|| std::fs::read_to_string(&files.ca).unwrap()), - ca_path: matches!(ca, Ca::TrustedDirectory).then(|| directory_holding(&files.ca)), - certfile: client_certificate.then(|| files.client_cert.clone()), - keyfile: client_certificate.then(|| files.client_key.clone()), - verify_certificate, - check_hostname, - }; - let client = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { + .unwrap() +} + +fn tls_client( + server: &litellm_redis_testing::RedisTlsServer, + host: &str, + tls: TlsOptions, +) -> Result { + litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { target: Target::Standalone(litellm_redis_client::Standalone { host: host.to_owned(), port: server.port(), @@ -71,8 +68,46 @@ async fn ping_over_tls( events: None, health_check: None, }) - .unwrap(); - client.ping().await +} + +fn tls_options( + files: &litellm_redis_testing::TlsFiles, + ca: Ca, + verify_certificate: bool, + check_hostname: bool, + client_key: ClientKey, +) -> TlsOptions { + let (keyfile, key_password) = match client_key { + ClientKey::Absent => (None, None), + ClientKey::Plain => (Some(files.client_key.clone()), None), + ClientKey::Encrypted { password } => ( + Some(files.encrypted_client_key.clone()), + password.map(SecretValue::new), + ), + }; + TlsOptions { + ca_certs: matches!(ca, Ca::Trusted).then(|| files.ca.clone()), + ca_data: matches!(ca, Ca::TrustedData).then(|| std::fs::read_to_string(&files.ca).unwrap()), + ca_path: matches!(ca, Ca::TrustedDirectory).then(|| directory_holding(&files.ca)), + certfile: keyfile.is_some().then(|| files.client_cert.clone()), + keyfile, + key_password, + verify_certificate, + check_hostname, + } +} + +async fn ping_over_tls( + require_client_certificate: bool, + host: &str, + ca: Ca, + verify_certificate: bool, + check_hostname: bool, + client_key: ClientKey, +) -> Result<(), litellm_redis_client::Error> { + let server = tls_server(require_client_certificate).await; + let tls = tls_options(server.files(), ca, verify_certificate, check_hostname, client_key); + tls_client(&server, host, tls).unwrap().ping().await } #[rstest] @@ -103,7 +138,15 @@ async fn tls_options_decide_which_servers_are_trusted( #[case] check_hostname: bool, #[case] connects: bool, ) { - let result = ping_over_tls(false, host, ca, verify_certificate, check_hostname, false).await; + let result = ping_over_tls( + false, + host, + ca, + verify_certificate, + check_hostname, + ClientKey::Absent, + ) + .await; assert_eq!(result.is_ok(), connects, "{result:?}"); if let Err(error) = result { @@ -115,24 +158,46 @@ async fn tls_options_decide_which_servers_are_trusted( } #[rstest] -#[case::with_the_client_certificate(true, true)] -#[case::without_a_client_certificate(false, false)] +#[case::with_the_client_certificate(ClientKey::Plain, true)] +#[case::with_an_encrypted_key_and_its_ssl_password( + ClientKey::Encrypted { password: Some(litellm_redis_testing::CLIENT_KEY_PASSWORD) }, + true +)] +#[case::without_a_client_certificate(ClientKey::Absent, false)] #[tokio::test] async fn mutual_tls_sends_the_configured_client_certificate( - #[case] client_certificate: bool, + #[case] client_key: ClientKey, #[case] connects: bool, ) { - let result = ping_over_tls( - true, - "127.0.0.1", + let result = ping_over_tls(true, "127.0.0.1", Ca::Trusted, true, false, client_key).await; + + assert_eq!(result.is_ok(), connects, "{result:?}"); +} + +#[rstest] +#[case::without_ssl_password(None, "ssl_password is required")] +#[case::with_the_wrong_ssl_password(Some("wrong-password"), "does not decrypt the key")] +#[tokio::test] +async fn an_encrypted_key_that_cannot_be_decrypted_fails_before_connecting( + #[case] password: Option<&'static str>, + #[case] reason: &str, +) { + let server = tls_server(true).await; + let tls = tls_options( + server.files(), Ca::Trusted, true, false, - client_certificate, - ) - .await; + ClientKey::Encrypted { password }, + ); - assert_eq!(result.is_ok(), connects, "{result:?}"); + let error = tls_client(&server, "127.0.0.1", tls).err().unwrap(); + + let litellm_redis_client::ConnectError::TlsKey { path, reason: cause } = &error else { + panic!("expected a TLS key error, got {error:?}"); + }; + assert_eq!(*path, server.files().encrypted_client_key.display().to_string()); + assert!(cause.to_string().contains(reason), "{cause}"); } #[rstest] @@ -161,3 +226,49 @@ fn a_missing_tls_file_names_the_path() { assert!(error.to_string().contains("/nonexistent/ca.pem"), "{error}"); } + +#[rstest] +fn a_legacy_encrypted_pem_key_is_refused_with_a_conversion_hint() { + let directory = std::env::temp_dir().join(format!("litellm-legacy-key-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let keyfile = directory.join("client.key"); + std::fs::write( + &keyfile, + "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,00112233445566778899AABBCCDDEEFF\n\nAAAA\n-----END RSA PRIVATE KEY-----\n", + ) + .unwrap(); + let tls = TlsOptions { + certfile: Some(keyfile.clone()), + keyfile: Some(keyfile), + key_password: Some(SecretValue::new("legacy-password")), + ..TlsOptions::verified() + }; + + let error = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { + target: Target::Standalone(litellm_redis_client::Standalone { + host: "127.0.0.1".to_owned(), + port: 6379, + tls: Some(tls), + db: 0, + }), + credentials: Credentials::NONE, + namespace: litellm_redis_client::Namespace::default(), + timeouts: litellm_redis_client::Timeouts::default(), + breaker: None, + events: None, + health_check: None, + }) + .err() + .unwrap(); + + assert!( + matches!( + &error, + litellm_redis_client::ConnectError::TlsKey { + reason: litellm_redis_client::KeyError::LegacyEncryption, + .. + } + ), + "{error:?}" + ); +} diff --git a/litellm-rust/crates/redis-testing/Cargo.toml b/litellm-rust/crates/redis-testing/Cargo.toml index 18fefb273bc..df97617c769 100644 --- a/litellm-rust/crates/redis-testing/Cargo.toml +++ b/litellm-rust/crates/redis-testing/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +pkcs8 = { version = "0.10.2", features = ["encryption", "pem", "std"] } rcgen = "0.14.10" tempfile.workspace = true testcontainers-modules = { workspace = true, features = ["redis"] } diff --git a/litellm-rust/crates/redis-testing/src/error.rs b/litellm-rust/crates/redis-testing/src/error.rs index ac0a526a293..ef732379a68 100644 --- a/litellm-rust/crates/redis-testing/src/error.rs +++ b/litellm-rust/crates/redis-testing/src/error.rs @@ -12,6 +12,8 @@ pub enum Error { Command { command: String, code: i64 }, #[error("generating test TLS certificates")] Certificate(#[from] rcgen::Error), + #[error("encrypting the test client key")] + KeyEncryption(#[from] pkcs8::Error), #[error("writing test TLS files")] Files(#[source] io::Error), #[error("Sentinel did not promote the replica")] diff --git a/litellm-rust/crates/redis-testing/src/lib.rs b/litellm-rust/crates/redis-testing/src/lib.rs index 905d387200b..aee570ba560 100644 --- a/litellm-rust/crates/redis-testing/src/lib.rs +++ b/litellm-rust/crates/redis-testing/src/lib.rs @@ -12,4 +12,4 @@ pub use deployment::{ClientConnection, RedisDeployment, Topology, eventually}; pub use error::Error; pub use sentinel::{RedisSentinel, SERVICE_NAME}; pub use standalone::RedisServer; -pub use tls::{RedisTlsServer, SERVER_NAME, TlsFiles}; +pub use tls::{CLIENT_KEY_PASSWORD, RedisTlsServer, SERVER_NAME, TlsFiles}; diff --git a/litellm-rust/crates/redis-testing/src/tls.rs b/litellm-rust/crates/redis-testing/src/tls.rs index 65cb9b1c79b..f412a72e94a 100644 --- a/litellm-rust/crates/redis-testing/src/tls.rs +++ b/litellm-rust/crates/redis-testing/src/tls.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use pkcs8::{LineEnding, PrivateKeyInfo, pkcs5::pbes2}; use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, DnType, IsCa, KeyPair}; use tempfile::TempDir; use testcontainers_modules::testcontainers::{ @@ -15,12 +16,15 @@ use crate::{ }; pub const SERVER_NAME: &str = "localhost"; +pub const CLIENT_KEY_PASSWORD: &str = "redis-testing-key-password"; pub struct TlsFiles { _directory: TempDir, pub ca: PathBuf, pub client_cert: PathBuf, pub client_key: PathBuf, + /// `client_key` as a PKCS#8 `ENCRYPTED PRIVATE KEY` under `CLIENT_KEY_PASSWORD` + pub encrypted_client_key: PathBuf, } struct Generated { @@ -117,6 +121,7 @@ fn generate() -> Result { let ca_path = write("ca.pem", ca.pem())?; let client_cert = write("client.pem", client.pem())?; let client_key_path = write("client.key", client_key.serialize_pem())?; + let encrypted_client_key = write("client-encrypted.key", encrypt(&client_key)?)?; Ok(Generated { ca_pem: ca.pem(), server_cert: server.pem(), @@ -126,6 +131,16 @@ fn generate() -> Result { ca: ca_path, client_cert, client_key: client_key_path, + encrypted_client_key, }, }) } + +// The parameters `openssl pkcs8 -topk8` uses by default: PBES2 with PBKDF2-HMAC-SHA256 and AES-256-CBC +fn encrypt(key: &KeyPair) -> Result { + let der = key.serialize_der(); + let params = pbes2::Parameters::pbkdf2_sha256_aes256cbc(2048, b"redis-testing-salt", &[7; 16])?; + let encrypted = + PrivateKeyInfo::try_from(der.as_slice())?.encrypt_with_params(params, CLIENT_KEY_PASSWORD)?; + Ok(encrypted.to_pem("ENCRYPTED PRIVATE KEY", LineEnding::LF)?.to_string()) +} diff --git a/litellm-rust/crates/redis-types/src/lib.rs b/litellm-rust/crates/redis-types/src/lib.rs index d5419fb9425..424eb4f0b8c 100644 --- a/litellm-rust/crates/redis-types/src/lib.rs +++ b/litellm-rust/crates/redis-types/src/lib.rs @@ -111,6 +111,8 @@ pub struct TlsOptions { pub ca_path: Option, pub certfile: Option, pub keyfile: Option, + /// Unlocks an encrypted `keyfile`, redis-py's `ssl_password` + pub key_password: Option, pub verify_certificate: bool, pub check_hostname: bool, } @@ -123,6 +125,7 @@ impl TlsOptions { ca_path: None, certfile: None, keyfile: None, + key_password: None, verify_certificate: true, check_hostname: false, } diff --git a/tests/unit/test_redis.py b/tests/unit/test_redis.py index b33b2886327..9cba2a02e7d 100644 --- a/tests/unit/test_redis.py +++ b/tests/unit/test_redis.py @@ -2136,6 +2136,17 @@ def _expected_view(fields: dict[str, object]) -> dict[str, object]: }, id="ssl_ca_path_is_unsupported", ), + pytest.param( + {"host": "redis", "ssl": True, "ssl_certfile": "/client.pem", "ssl_password": "key-secret"}, + {}, + { + "rejected": { + "message": "Redis.__init__() got an unexpected keyword argument 'ssl_password'", + "type": "TypeError", + } + }, + id="ssl_password_is_unsupported", + ), pytest.param( { "db": 2,